cognite-toolkit 0.6.97__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py

@@ -1,29 +1,39 @@
-from collections.abc import Iterator, Mapping, Sequence
+from collections.abc import Iterable, Iterator, Mapping, Sequence
 from typing import ClassVar, cast
 
-from cognite.client.data_classes.data_modeling import InstanceApply, NodeId
+from cognite.client.data_classes import Annotation
+from cognite.client.data_classes.data_modeling import EdgeId, InstanceApply, NodeId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.pending_instances_ids import PendingInstanceId
 from cognite_toolkit._cdf_tk.constants import MISSING_EXTERNAL_ID, MISSING_INSTANCE_SPACE
-from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
+from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError, ToolkitValueError
 from cognite_toolkit._cdf_tk.storageio import (
+    AnnotationIO,
     HierarchyIO,
     InstanceIO,
     UploadableStorageIO,
 )
 from cognite_toolkit._cdf_tk.storageio._base import Page, UploadItem
+from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning
 from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, HTTPMessage, ItemsRequest, SuccessResponseItems
 from cognite_toolkit._cdf_tk.utils.useful_types import (
-    AssetCentricKind,
+    AssetCentricKindExtended,
     AssetCentricType,
     JsonVal,
     T_AssetCentricResource,
 )
 
-from .data_classes import AssetCentricMapping, AssetCentricMappingList, MigrationMapping, MigrationMappingList
+from .data_classes import (
+    AnnotationMapping,
+    AssetCentricMapping,
+    AssetCentricMappingList,
+    MigrationMapping,
+    MigrationMappingList,
+)
 from .data_model import INSTANCE_SOURCE_VIEW_ID
+from .default_mappings import ASSET_ANNOTATIONS_ID, FILE_ANNOTATIONS_ID
 from .selectors import AssetCentricMigrationSelector, MigrateDataSetSelector, MigrationCSVFileSelector
 
 
@@ -37,7 +47,7 @@ class AssetCentricMigrationIO(
     CHUNK_SIZE = 1000
     UPLOAD_ENDPOINT = InstanceIO.UPLOAD_ENDPOINT
 
-    PENDING_INSTANCE_ID_ENDPOINT_BY_KIND: ClassVar[Mapping[AssetCentricKind, str]] = {
+    PENDING_INSTANCE_ID_ENDPOINT_BY_KIND: ClassVar[Mapping[AssetCentricKindExtended, str]] = {
         "TimeSeries": "/timeseries/set-pending-instance-ids",
         "FileMetadata": "/files/set-pending-instance-ids",
     }
@@ -113,8 +123,8 @@ class AssetCentricMigrationIO(
             yield mapping_list
 
     @staticmethod
-    def _kind_to_resource_type(kind: AssetCentricKind) -> AssetCentricType:
-        mapping: dict[AssetCentricKind, AssetCentricType] = {
+    def _kind_to_resource_type(kind: AssetCentricKindExtended) -> AssetCentricType:
+        mapping: dict[AssetCentricKindExtended, AssetCentricType] = {
             "Assets": "asset",
             "Events": "event",
             "TimeSeries": "timeseries",
@@ -199,3 +209,142 @@ class AssetCentricMigrationIO(
             pending_instance_id=NodeId(item.space, item.external_id),
             id=id_,
         )
+
+
+class AnnotationMigrationIO(
+    UploadableStorageIO[AssetCentricMigrationSelector, AssetCentricMapping[Annotation], InstanceApply]
+):
+    """IO class for migrating Annotations.
+
+    Args:
+        client: The ToolkitClient to use for CDF interactions.
+        instance_space: The instance space to use for the migrated annotations.
+        default_asset_annotation_mapping: The default ingestion mapping to use for asset-linked annotations.
+        default_file_annotation_mapping: The default ingestion mappingto use for file-linked annotations.
+
+    """
+
+    KIND = "AnnotationMigration"
+    SUPPORTED_DOWNLOAD_FORMATS = frozenset({".parquet", ".csv", ".ndjson"})
+    SUPPORTED_COMPRESSIONS = frozenset({".gz"})
+    SUPPORTED_READ_FORMATS = frozenset({".parquet", ".csv", ".ndjson", ".yaml", ".yml"})
+    CHUNK_SIZE = 1000
+    UPLOAD_ENDPOINT = InstanceIO.UPLOAD_ENDPOINT
+
+    SUPPORTED_ANNOTATION_TYPES = frozenset({"diagrams.AssetLink", "diagrams.FileLink"})
+
+    def __init__(
+        self,
+        client: ToolkitClient,
+        instance_space: str | None = None,
+        default_asset_annotation_mapping: str | None = None,
+        default_file_annotation_mapping: str | None = None,
+    ) -> None:
+        super().__init__(client)
+        self.annotation_io = AnnotationIO(client)
+        self.instance_space = instance_space
+        self.default_asset_annotation_mapping = default_asset_annotation_mapping or ASSET_ANNOTATIONS_ID
+        self.default_file_annotation_mapping = default_file_annotation_mapping or FILE_ANNOTATIONS_ID
+
+    def as_id(self, item: AssetCentricMapping[Annotation]) -> str:
+        return f"Annotation_{item.mapping.id}"
+
+    def count(self, selector: AssetCentricMigrationSelector) -> int | None:
+        if isinstance(selector, MigrationCSVFileSelector):
+            return len(selector.items)
+        else:
+            # There is no efficient way to count annotations in CDF.
+            return None
+
+    def stream_data(self, selector: AssetCentricMigrationSelector, limit: int | None = None) -> Iterable[Page]:
+        if isinstance(selector, MigrateDataSetSelector):
+            iterator = self._stream_from_dataset(selector, limit)
+        elif isinstance(selector, MigrationCSVFileSelector):
+            iterator = self._stream_from_csv(selector, limit)
+        else:
+            raise ToolkitNotImplementedError(f"Selector {type(selector)} is not supported for stream_data")
+        yield from (Page(worker_id="main", items=items) for items in iterator)
+
+    def _stream_from_dataset(
+        self, selector: MigrateDataSetSelector, limit: int | None = None
+    ) -> Iterator[Sequence[AssetCentricMapping[Annotation]]]:
+        if self.instance_space is None:
+            raise ToolkitValueError("Instance space must be provided for dataset-based annotation migration.")
+        asset_centric_selector = selector.as_asset_centric_selector()
+        for data_chunk in self.annotation_io.stream_data(asset_centric_selector, limit):
+            mapping_list = AssetCentricMappingList[Annotation]([])
+            for resource in data_chunk.items:
+                if resource.annotation_type not in self.SUPPORTED_ANNOTATION_TYPES:
+                    # This should not happen, as the annotation_io should already filter these out.
+                    # This is just in case.
+                    continue
+                mapping = AnnotationMapping(
+                    instance_id=EdgeId(space=self.instance_space, external_id=f"annotation_{resource.id!r}"),
+                    id=resource.id,
+                    ingestion_view=self._get_mapping(selector.ingestion_mapping, resource),
+                    preferred_consumer_view=selector.preferred_consumer_view,
+                    # The PySDK is poorly typed.
+                    annotation_type=resource.annotation_type,  # type: ignore[arg-type]
+                )
+                mapping_list.append(AssetCentricMapping(mapping=mapping, resource=resource))
+            yield mapping_list
+
+    def _stream_from_csv(
+        self, selector: MigrationCSVFileSelector, limit: int | None = None
+    ) -> Iterator[Sequence[AssetCentricMapping[Annotation]]]:
+        items = selector.items
+        if limit is not None:
+            items = MigrationMappingList(items[:limit])
+        chunk: list[AssetCentricMapping[Annotation]] = []
+        for current_batch in chunker_sequence(items, self.CHUNK_SIZE):
+            resources = self.client.annotations.retrieve_multiple(current_batch.get_ids())
+            resources_by_id = {resource.id: resource for resource in resources}
+            not_found = 0
+            incorrect_type_count = 0
+            for mapping in current_batch:
+                resource = resources_by_id.get(mapping.id)
+                if resource is None:
+                    not_found += 1
+                    continue
+                if resource.annotation_type not in self.SUPPORTED_ANNOTATION_TYPES:
+                    incorrect_type_count += 1
+                    continue
+                mapping.ingestion_view = self._get_mapping(mapping.ingestion_view, resource)
+                chunk.append(AssetCentricMapping(mapping=mapping, resource=resource))
+            if chunk:
+                yield chunk
+                chunk = []
+            if not_found:
+                MediumSeverityWarning(
+                    f"Could not find {not_found} annotations referenced in the CSV file. They will be skipped during migration."
+                ).print_warning(include_timestamp=True, console=self.client.console)
+            if incorrect_type_count:
+                MediumSeverityWarning(
+                    f"Found {incorrect_type_count} annotations with unsupported types. Only 'diagrams.AssetLink' and "
+                    "'diagrams.FileLink' are supported. These annotations will be skipped during migration."
+                ).print_warning(include_timestamp=True, console=self.client.console)
+
+    def _get_mapping(self, current_mapping: str | None, resource: Annotation) -> str:
+        try:
+            return (
+                current_mapping
+                or {
+                    "diagrams.AssetLink": self.default_asset_annotation_mapping,
+                    "diagrams.FileLink": self.default_file_annotation_mapping,
+                }[resource.annotation_type]
+            )
+        except KeyError as e:
+            raise ToolkitValueError(
+                f"Could not determine default ingestion view for annotation type '{resource.annotation_type}'. "
+                "Please specify the ingestion view explicitly in the CSV file."
+            ) from e
+
+    def json_to_resource(self, item_json: dict[str, JsonVal]) -> InstanceApply:
+        raise NotImplementedError("Deserializing Annotation Migrations from JSON is not supported.")
+
+    def data_to_json_chunk(
+        self,
+        data_chunk: Sequence[AssetCentricMapping[Annotation]],
+        selector: AssetCentricMigrationSelector | None = None,
+    ) -> list[dict[str, JsonVal]]:
+        raise NotImplementedError("Serializing Annotation Migrations to JSON is not supported.")

cognite_toolkit/_cdf_tk/commands/_migrate/selectors.py

@@ -8,11 +8,11 @@ from cognite.client.data_classes.data_modeling import ViewId
 from cognite_toolkit._cdf_tk.commands._migrate.data_classes import MigrationMappingList
 from cognite_toolkit._cdf_tk.storageio import DataSelector
 from cognite_toolkit._cdf_tk.storageio.selectors import DataSetSelector
-from cognite_toolkit._cdf_tk.utils.useful_types import AssetCentricKind
+from cognite_toolkit._cdf_tk.utils.useful_types import AssetCentricKindExtended
 
 
 class AssetCentricMigrationSelector(DataSelector, ABC):
-    kind: AssetCentricKind
+    kind: AssetCentricKindExtended
 
     @abstractmethod
     def get_ingestion_mappings(self) -> list[str]:
@@ -41,7 +41,7 @@ class MigrationCSVFileSelector(AssetCentricMigrationSelector):
 
 class MigrateDataSetSelector(AssetCentricMigrationSelector):
     type: Literal["migrateDataSet"] = "migrateDataSet"
-    kind: AssetCentricKind
+    kind: AssetCentricKindExtended
     data_set_external_id: str
     ingestion_mapping: str | None = None
     preferred_consumer_view: ViewId | None = None
@@ -63,4 +63,9 @@ class MigrateDataSetSelector(AssetCentricMigrationSelector):
         return [self.ingestion_mapping] if self.ingestion_mapping else []
 
     def as_asset_centric_selector(self) -> DataSetSelector:
-        return DataSetSelector(data_set_external_id=self.data_set_external_id, kind=self.kind)
+        if self.kind == "Annotations":
+            # Annotations are connected to file metadata, so we need to download the file metadata
+            # and look up the annotations connected to each file metadata.
+            return DataSetSelector(data_set_external_id=self.data_set_external_id, kind="FileMetadata")
+        else:
+            return DataSetSelector(data_set_external_id=self.data_set_external_id, kind=self.kind)

cognite_toolkit/_cdf_tk/commands/_purge.py

@@ -7,12 +7,7 @@ from typing import Literal, cast
 
 import questionary
 from cognite.client.data_classes import DataSetUpdate
-from cognite.client.data_classes._base import CogniteResourceList
-from cognite.client.data_classes.data_modeling import (
-    EdgeList,
-    NodeId,
-    NodeList,
-)
+from cognite.client.data_classes.data_modeling import Edge, NodeId
 from cognite.client.data_classes.data_modeling.statistics import SpaceStatistics
 from cognite.client.exceptions import CogniteAPIError
 from cognite.client.utils._identifier import InstanceId
@@ -46,6 +41,7 @@ from cognite_toolkit._cdf_tk.exceptions import (
     AuthorizationError,
     ToolkitMissingResourceError,
 )
+from cognite_toolkit._cdf_tk.protocols import ResourceResponseProtocol
 from cognite_toolkit._cdf_tk.storageio import InstanceIO
 from cognite_toolkit._cdf_tk.storageio.selectors import InstanceSelector
 from cognite_toolkit._cdf_tk.tk_warnings import (
@@ -107,7 +103,7 @@ class ToDelete(ABC):
     @abstractmethod
     def get_process_function(
         self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
-    ) -> Callable[[CogniteResourceList], list[JsonVal]]:
+    ) -> Callable[[list[ResourceResponseProtocol]], list[JsonVal]]:
         raise NotImplementedError()
 
     def get_extra_fields(self) -> dict[str, JsonVal]:
@@ -118,9 +114,10 @@ class ToDelete(ABC):
 class DataModelingToDelete(ToDelete):
     def get_process_function(
         self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
-    ) -> Callable[[CogniteResourceList], list[JsonVal]]:
-        def as_id(chunk: CogniteResourceList) -> list[JsonVal]:
-            return [item.as_id().dump(include_type=False) for item in chunk]
+    ) -> Callable[[list[ResourceResponseProtocol]], list[JsonVal]]:
+        def as_id(chunk: list[ResourceResponseProtocol]) -> list[JsonVal]:
+            # We know that all data modeling resources implement as_id
+            return [item.as_id().dump(include_type=False) for item in chunk]  # type: ignore[attr-defined]
 
         return as_id
 
@@ -129,11 +126,11 @@ class DataModelingToDelete(ToDelete):
 class EdgeToDelete(ToDelete):
     def get_process_function(
         self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
-    ) -> Callable[[CogniteResourceList], list[JsonVal]]:
-        def as_id(chunk: CogniteResourceList) -> list[JsonVal]:
+    ) -> Callable[[list[ResourceResponseProtocol]], list[JsonVal]]:
+        def as_id(chunk: list[ResourceResponseProtocol]) -> list[JsonVal]:
             return [
                 {"space": item.space, "externalId": item.external_id, "instanceType": "edge"}
-                for item in cast(EdgeList, chunk)
+                for item in cast(list[Edge], chunk)
             ]
 
         return as_id
@@ -146,9 +143,10 @@ class NodesToDelete(ToDelete):
 
     def get_process_function(
         self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
-    ) -> Callable[[CogniteResourceList], list[JsonVal]]:
-        def check_for_data(chunk: CogniteResourceList) -> list[JsonVal]:
-            node_ids = cast(NodeList, chunk).as_ids()
+    ) -> Callable[[list[ResourceResponseProtocol]], list[JsonVal]]:
+        def check_for_data(chunk: list[ResourceResponseProtocol]) -> list[JsonVal]:
+            # We know that all node resources implement as_id
+            node_ids = [item.as_id() for item in chunk]  # type: ignore[attr-defined]
             found_ids: set[InstanceId] = set()
             if not self.delete_datapoints:
                 timeseries = client.time_series.retrieve_multiple(instance_ids=node_ids, ignore_unknown_ids=True)
@@ -164,8 +162,7 @@ class NodesToDelete(ToDelete):
                 dumped = node_id.dump(include_instance_type=True)
                 # The delete endpoint expects "instanceType" instead of "type"
                 dumped["instanceType"] = dumped.pop("type")
-                # MyPy think complains about invariant here, even though dict[str, str] is a type of JsonVal
-                result.append(dumped)  # type: ignore[arg-type]
+                result.append(dumped)
             return result
 
         return check_for_data
@@ -175,9 +172,10 @@ class NodesToDelete(ToDelete):
 class IdResourceToDelete(ToDelete):
     def get_process_function(
         self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
-    ) -> Callable[[CogniteResourceList], list[JsonVal]]:
-        def as_id(chunk: CogniteResourceList) -> list[JsonVal]:
-            return [{"id": item.id} for item in chunk]
+    ) -> Callable[[list[ResourceResponseProtocol]], list[JsonVal]]:
+        def as_id(chunk: list[ResourceResponseProtocol]) -> list[JsonVal]:
+            # We know that all id resources have an id attribute
+            return [{"id": item.id} for item in chunk]  # type: ignore[attr-defined]
 
         return as_id
 
@@ -186,9 +184,10 @@ class IdResourceToDelete(ToDelete):
 class ExternalIdToDelete(ToDelete):
     def get_process_function(
         self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
-    ) -> Callable[[CogniteResourceList], list[JsonVal]]:
-        def as_external_id(chunk: CogniteResourceList) -> list[JsonVal]:
-            return [{"externalId": item.external_id} for item in chunk]
+    ) -> Callable[[list[ResourceResponseProtocol]], list[JsonVal]]:
+        def as_external_id(chunk: list[ResourceResponseProtocol]) -> list[JsonVal]:
+            # We know that all external id resources have an external_id attribute
+            return [{"externalId": item.external_id} for item in chunk]  # type: ignore[attr-defined]
 
         return as_external_id
 
@@ -321,7 +320,7 @@ class PurgeCommand(ToolkitCommand):
                     iteration_count = item.total // self.BATCH_SIZE_DM + (
                         1 if item.total % self.BATCH_SIZE_DM > 0 else 0
                     )
-                executor = ProducerWorkerExecutor[CogniteResourceList, list[JsonVal]](
+                executor = ProducerWorkerExecutor[list[ResourceResponseProtocol], list[JsonVal]](
                     download_iterable=self._iterate_batch(
                         item.crud, space, data_set_external_id, batch_size=self.BATCH_SIZE_DM
                     ),
@@ -348,13 +347,13 @@ class PurgeCommand(ToolkitCommand):
     @staticmethod
     def _iterate_batch(
         crud: ResourceCRUD, selected_space: str | None, data_set_external_id: str | None, batch_size: int
-    ) -> Iterable[CogniteResourceList]:
-        batch = crud.list_cls([])
+    ) -> Iterable[list[ResourceResponseProtocol]]:
+        batch: list[ResourceResponseProtocol] = []
         for resource in crud.iterate(space=selected_space, data_set_external_id=data_set_external_id):
             batch.append(resource)
             if len(batch) >= batch_size:
                 yield batch
-                batch = crud.list_cls([])
+                batch = []
         if batch:
             yield batch
 

cognite_toolkit/_cdf_tk/commands/_questionary_style.py

@@ -0,0 +1,16 @@
+import questionary
+
+custom_style_fancy = questionary.Style(
+    [
+        ("qmark", "fg:#673ab7"),  # token in front of the question
+        ("question", "bold"),  # question text
+        ("answer", "fg:#f44336 bold"),  # submitted answer text behind the question
+        ("pointer", "fg:#673ab7 bold"),  # pointer used in select and checkbox prompts
+        ("highlighted", "fg:#673ab7 bold"),  # pointed-at choice in select and checkbox prompts
+        ("selected", "fg:#673ab7"),  # style for a selected item of a checkbox
+        ("separator", "fg:#cc5454"),  # separator in lists
+        ("instruction", ""),  # user instructions for select, rawselect, checkbox
+        ("text", ""),  # plain text
+        ("disabled", "fg:#858585 italic"),  # disabled choices for select and checkbox prompts
+    ]
+)

cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -1,32 +1,33 @@
+from collections import Counter
 from collections.abc import Sequence
 from functools import partial
 from pathlib import Path
 
-from cognite.client.data_classes._base import T_CogniteResource
-from pydantic import ValidationError
+from cognite.client.data_classes.data_modeling import (
+    ViewId,
+)
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM, DATA_RESOURCE_DIR
+from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_SUFFIX, DATA_RESOURCE_DIR
+from cognite_toolkit._cdf_tk.cruds import ViewCRUD
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
+from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.storageio import (
     T_Selector,
     UploadableStorageIO,
-    are_same_kind,
     get_upload_io,
 )
-from cognite_toolkit._cdf_tk.storageio._base import T_WriteCogniteResource, TableUploadableStorageIO, UploadItem
-from cognite_toolkit._cdf_tk.storageio.selectors import Selector, SelectorAdapter
-from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
-from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+from cognite_toolkit._cdf_tk.storageio._base import TableUploadableStorageIO, UploadItem
+from cognite_toolkit._cdf_tk.storageio.selectors import Selector, load_selector
+from cognite_toolkit._cdf_tk.storageio.selectors._instances import InstanceSpaceSelector
+from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning, ToolkitWarning
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
-from cognite_toolkit._cdf_tk.utils.file import read_yaml_file
-from cognite_toolkit._cdf_tk.utils.fileio import TABLE_READ_CLS_BY_FORMAT, FileReader
+from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, ItemMessage, SuccessResponseItems
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
 from cognite_toolkit._cdf_tk.utils.progress_tracker import ProgressTracker
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
-from cognite_toolkit._cdf_tk.validation import humanize_validation_error
 
 from ._base import ToolkitCommand
 from .deploy import DeployCommand
@@ -76,53 +77,76 @@ class UploadCommand(ToolkitCommand):
         │   │   └── table2.Table.yaml
         │   └── ...
         ├── datafile1.kind.ndjson # Data file of a specific kind
-        ├── datafile1.Metadata.yaml       # Metadata file for datafile1
+        ├── datafile1.Manifest.yaml       # Manifest for datafile1
         ├── datafile2.kind2.ndjson # Another data file of the same or different kind
-        ├── datafile2.Metadata.yaml       # Metadata file for datafile2
+        ├── datafile2.Manifest.yaml       # Manifest file for datafile2
         └── ...
         """
-        console = Console()
-        data_files_by_selector = self._find_data_files(input_dir, kind)
+        console = client.console
+        data_files_by_selector = self._find_data_files(input_dir)
 
         self._deploy_resource_folder(input_dir / DATA_RESOURCE_DIR, deploy_resources, client, console, dry_run, verbose)
 
+        data_files_by_selector = self._topological_sort_if_instance_selector(data_files_by_selector, client)
+
         self._upload_data(data_files_by_selector, client, dry_run, input_dir, console, verbose)
 
+    def _topological_sort_if_instance_selector(
+        self, data_files_by_selector: dict[Selector, list[Path]], client: ToolkitClient
+    ) -> dict[Selector, list[Path]]:
+        """Topologically sorts InstanceSpaceSelectors (if they are present) to determine the order of upload based on container dependencies from the views.
+
+        Args:
+            data_files_by_selector: A dictionary mapping selectors to their data files.
+            client: The cognite client to use for the upload.
+
+        Returns:
+            A dictionary mapping selectors to their data files with necessary preprocessing.
+        """
+        counts = Counter(type(selector) for selector in data_files_by_selector.keys())
+        if counts[InstanceSpaceSelector] <= 1:
+            return data_files_by_selector
+
+        selector_by_view_id: dict[ViewId, Selector] = {}
+        for selector in data_files_by_selector:
+            if isinstance(selector, InstanceSpaceSelector) and selector.view is not None:
+                selector_by_view_id[selector.view.as_id()] = selector
+
+        view_dependencies = ViewCRUD.create_loader(client).topological_sort_container_constraints(
+            list(selector_by_view_id.keys())
+        )
+        prepared_selectors: dict[Selector, list[Path]] = {}
+
+        # Reorder selectors according to the dependency-sorted view list
+        for view_id in view_dependencies:
+            selector = selector_by_view_id[view_id]
+            prepared_selectors[selector] = data_files_by_selector[selector]
+
+        # Preserve selectors that aren't affected by view dependencies
+        # (e.g., raw tables, time series, non-view instance data)
+        for selector in data_files_by_selector.keys():
+            if selector not in prepared_selectors:
+                prepared_selectors[selector] = data_files_by_selector[selector]
+
+        return prepared_selectors
+
     def _find_data_files(
         self,
         input_dir: Path,
-        kind: str | None = None,
     ) -> dict[Selector, list[Path]]:
         """Finds data files and their corresponding metadata files in the input directory."""
-        manifest_file_endswith = f".{DATA_MANIFEST_STEM}.yaml"
         data_files_by_metadata: dict[Selector, list[Path]] = {}
-        for metadata_file in input_dir.glob(f"*{manifest_file_endswith}"):
-            data_file_prefix = metadata_file.name.removesuffix(manifest_file_endswith)
-            data_files = [
-                file
-                for file in input_dir.glob(f"{data_file_prefix}*")
-                if not file.name.endswith(manifest_file_endswith)
-            ]
-            if kind is not None and data_files:
-                data_files = [data_file for data_file in data_files if are_same_kind(kind, data_file)]
-                if not data_files:
-                    continue
+        for manifest_file in input_dir.glob(f"*{DATA_MANIFEST_SUFFIX}"):
+            selector_or_warning = load_selector(manifest_file)
+            if isinstance(selector_or_warning, ToolkitWarning):
+                self.warn(selector_or_warning)
+                continue
+            selector: Selector = selector_or_warning
+            data_files = selector.find_data_files(input_dir, manifest_file)
             if not data_files:
                 self.warn(
                     MediumSeverityWarning(
-                        f"Metadata file {metadata_file.as_posix()!r} has no corresponding data files, skipping.",
-                    )
-                )
-                continue
-
-            selector_dict = read_yaml_file(metadata_file, expected_output="dict")
-            try:
-                selector = SelectorAdapter.validate_python(selector_dict)
-            except ValidationError as e:
-                errors = humanize_validation_error(e)
-                self.warn(
-                    ResourceFormatWarning(
-                        metadata_file, tuple(errors), text="Invalid selector in metadata file, skipping."
+                        f"Metadata file {manifest_file.as_posix()!r} has no corresponding data files, skipping.",
                     )
                 )
                 continue
@@ -178,49 +202,48 @@ class UploadCommand(ToolkitCommand):
                 io = self._create_selected_io(selector, datafiles[0], client)
                 if io is None:
                     continue
-                for data_file in datafiles:
-                    file_display = self._path_as_display_name(data_file)
-                    if verbose:
-                        console.print(f"{action} {selector.display_name} from {file_display.as_posix()!r}")
-                    reader = FileReader.from_filepath(data_file)
-                    is_table = reader.format in TABLE_READ_CLS_BY_FORMAT
-                    if is_table and not isinstance(io, TableUploadableStorageIO):
-                        raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
-                    tracker = ProgressTracker[str]([self._UPLOAD])
-                    executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
-                        download_iterable=io.read_chunks(reader),
-                        process=partial(io.rows_to_data, selector=selector)
-                        if is_table and isinstance(io, TableUploadableStorageIO)
-                        else io.json_chunk_to_data,
-                        write=partial(
-                            self._upload_items,
-                            upload_client=upload_client,
-                            io=io,
-                            dry_run=dry_run,
-                            selector=selector,
-                            tracker=tracker,
-                            console=console,
-                        ),
-                        iteration_count=None,
-                        max_queue_size=self._MAX_QUEUE_SIZE,
-                        download_description=f"Reading {file_count:,}/{total_file_count + 1:,}: {file_display.as_posix()!s}",
-                        process_description="Processing",
-                        write_description=f"{action} {selector.display_name!r}",
+                reader = MultiFileReader(datafiles)
+                if reader.is_table and not isinstance(io, TableUploadableStorageIO):
+                    raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
+
+                chunk_count = io.count_chunks(reader)
+                iteration_count = chunk_count // io.CHUNK_SIZE + (1 if chunk_count % io.CHUNK_SIZE > 0 else 0)
+
+                tracker = ProgressTracker[str]([self._UPLOAD])
+                executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
+                    download_iterable=io.read_chunks(reader, selector),
+                    process=partial(io.rows_to_data, selector=selector)
+                    if reader.is_table and isinstance(io, TableUploadableStorageIO)
+                    else io.json_chunk_to_data,
+                    write=partial(
+                        self._upload_items,
+                        upload_client=upload_client,
+                        io=io,
+                        dry_run=dry_run,
+                        selector=selector,
+                        tracker=tracker,
                         console=console,
-                    )
-                    executor.run()
-                    file_count += 1
-                    executor.raise_on_error()
-                    final_action = "Uploaded" if not dry_run else "Would upload"
-                    suffix = " successfully" if not dry_run else ""
-                    results = tracker.aggregate()
-                    success = results.get((self._UPLOAD, "success"), 0)
-                    failed = results.get((self._UPLOAD, "failed"), 0)
-                    if failed > 0:
-                        suffix += f", {failed:,} failed"
-                    console.print(
-                        f"{final_action} {success:,} {selector.display_name} from {file_display.as_posix()!r}{suffix}."
-                    )
+                    ),
+                    iteration_count=iteration_count,
+                    max_queue_size=self._MAX_QUEUE_SIZE,
+                    download_description=f"Reading {selector.display_name!r} files",
+                    process_description="Processing",
+                    write_description=f"{action} {selector.display_name!r}",
+                    console=console,
+                )
+                executor.run()
+                file_count += len(datafiles)
+                executor.raise_on_error()
+                final_action = "Uploaded" if not dry_run else "Would upload"
+                suffix = " successfully" if not dry_run else ""
+                results = tracker.aggregate()
+                success = results.get((self._UPLOAD, "success"), 0)
+                failed = results.get((self._UPLOAD, "failed"), 0)
+                if failed > 0:
+                    suffix += f", {failed:,} failed"
+                console.print(
+                    f"{final_action} {success:,} {selector.display_name} from {len(datafiles)} files{suffix}."
+                )
 
     @staticmethod
     def _path_as_display_name(input_path: Path, cwd: Path = Path.cwd()) -> Path:
@@ -233,7 +256,7 @@ class UploadCommand(ToolkitCommand):
         self, selector: Selector, data_file: Path, client: ToolkitClient
     ) -> UploadableStorageIO | None:
         try:
-            io_cls = get_upload_io(type(selector), kind=data_file)
+            io_cls = get_upload_io(selector)
         except ValueError as e:
             self.warn(HighSeverityWarning(f"Could not find StorageIO for selector {selector}: {e}"))
             return None
@@ -242,9 +265,9 @@ class UploadCommand(ToolkitCommand):
     @classmethod
     def _upload_items(
         cls,
-        data_chunk: Sequence[UploadItem],
+        data_chunk: Sequence[UploadItem[T_ResourceRequest]],
         upload_client: HTTPClient,
-        io: UploadableStorageIO[T_Selector, T_CogniteResource, T_WriteCogniteResource],
+        io: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
         selector: T_Selector,
         dry_run: bool,
         tracker: ProgressTracker[str],