cognite-toolkit 0.7.25__py3-none-any.whl → 0.7.27__py3-none-any.whl

cognite_toolkit/_cdf_tk/apps/_migrate_app.py

@@ -8,17 +8,14 @@ from cognite.client.data_classes import Annotation
 from cognite.client.data_classes.data_modeling import ContainerId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.commands import (
-    MigrationCanvasCommand,
-    MigrationPrepareCommand,
-)
+from cognite_toolkit._cdf_tk.commands import MigrationPrepareCommand
 from cognite_toolkit._cdf_tk.commands._migrate import MigrationCommand
 from cognite_toolkit._cdf_tk.commands._migrate.creators import (
     InfieldV2ConfigCreator,
     InstanceSpaceCreator,
     SourceSystemCreator,
 )
-from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper, ChartMapper
+from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper, CanvasMapper, ChartMapper
 from cognite_toolkit._cdf_tk.commands._migrate.migration_io import (
     AnnotationMigrationIO,
     AssetCentricMigrationIO,
@@ -28,14 +25,15 @@ from cognite_toolkit._cdf_tk.commands._migrate.selectors import (
     MigrateDataSetSelector,
     MigrationCSVFileSelector,
 )
-from cognite_toolkit._cdf_tk.storageio import ChartIO
-from cognite_toolkit._cdf_tk.storageio.selectors import ChartExternalIdSelector
+from cognite_toolkit._cdf_tk.storageio import CanvasIO, ChartIO
+from cognite_toolkit._cdf_tk.storageio.selectors import CanvasExternalIdSelector, ChartExternalIdSelector
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._cdf_tk.utils.cli_args import parse_view_str
 from cognite_toolkit._cdf_tk.utils.interactive_select import (
     AssetInteractiveSelect,
     DataModelingSelect,
     FileMetadataInteractiveSelect,
+    InteractiveCanvasSelect,
     InteractiveChartSelect,
     ResourceViewMappingInteractiveSelect,
 )
@@ -855,6 +853,23 @@ class MigrateApp(typer.Typer):
                 "performed to select the Canvas to migrate."
             ),
         ] = None,
+        skip_on_missing_ref: Annotated[
+            bool,
+            typer.Option(
+                "--skip-on-missing-ref",
+                "-s",
+                help="If set, the migration will skip Canvases that reference resources that have not been migrated to data modeling. "
+                "If not set, the migration will continue but the result will exclude the missing references.",
+            ),
+        ] = False,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where migration logs will be stored.",
+            ),
+        ] = Path(f"migration_logs_{TODAY}"),
         dry_run: Annotated[
             bool,
             typer.Option(
@@ -880,12 +895,24 @@ class MigrateApp(typer.Typer):
         is populated with the mapping from Asset-Centric resources to the new data modeling resources.
         """
         client = EnvironmentVariables.create_from_environment().get_client()
+        if external_id is None:
+            interactive = InteractiveCanvasSelect(client)
+            external_id = interactive.select_external_ids()
+            log_dir = questionary.path("Specify log directory for migration logs:", default=str(log_dir)).ask()
+            dry_run = questionary.confirm("Do you want to perform a dry run?", default=dry_run).ask()
+            verbose = questionary.confirm("Do you want verbose output?", default=verbose).ask()
+            if any(res is None for res in [log_dir, dry_run, verbose]):
+                raise typer.Abort()
+            log_dir = Path(log_dir)
 
-        cmd = MigrationCanvasCommand()
+        cmd = MigrationCommand()
+        selector = CanvasExternalIdSelector(external_ids=tuple(external_id))
         cmd.run(
-            lambda: cmd.migrate_canvas(
-                client,
-                external_ids=external_id,
+            lambda: cmd.migrate(
+                selected=selector,
+                data=CanvasIO(client, exclude_existing_version=True),
+                mapper=CanvasMapper(client, dry_run=dry_run, skip_on_missing_ref=skip_on_missing_ref),
+                log_dir=log_dir,
                 dry_run=dry_run,
                 verbose=verbose,
             )

cognite_toolkit/_cdf_tk/commands/__init__.py

@@ -1,6 +1,6 @@
 from ._download import DownloadCommand
 from ._migrate import (
-    MigrationCanvasCommand,
+    MigrationCommand,
     MigrationPrepareCommand,
 )
 from ._profile import ProfileAssetCentricCommand, ProfileAssetCommand, ProfileRawCommand, ProfileTransformationCommand
@@ -30,7 +30,7 @@ __all__ = [
     "DownloadCommand",
     "DumpResourceCommand",
     "InitCommand",
-    "MigrationCanvasCommand",
+    "MigrationCommand",
     "MigrationPrepareCommand",
     "ModulesCommand",
     "ProfileAssetCentricCommand",

cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py

@@ -1,9 +1,7 @@
-from .canvas import MigrationCanvasCommand
 from .command import MigrationCommand
 from .prepare import MigrationPrepareCommand
 
 __all__ = [
-    "MigrationCanvasCommand",
     "MigrationCommand",
     "MigrationPrepareCommand",
 ]

cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py

@@ -1,5 +1,6 @@
 from abc import ABC, abstractmethod
-from collections.abc import Sequence
+from collections import defaultdict
+from collections.abc import Callable, Sequence
 from typing import Generic, cast
 from uuid import uuid4
 
@@ -11,8 +12,15 @@ from cognite.client.data_classes.data_modeling import (
     View,
     ViewId,
 )
+from cognite.client.exceptions import CogniteException
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.client.data_classes.canvas import (
+    ContainerReferenceApply,
+    FdmInstanceContainerReferenceApply,
+    IndustrialCanvas,
+    IndustrialCanvasApply,
+)
 from cognite_toolkit._cdf_tk.client.data_classes.charts import Chart, ChartWrite
 from cognite_toolkit._cdf_tk.client.data_classes.charts_data import (
     ChartCoreTimeseries,
@@ -23,13 +31,18 @@ from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMa
 from cognite_toolkit._cdf_tk.commands._migrate.conversion import DirectRelationCache, asset_centric_to_dm
 from cognite_toolkit._cdf_tk.commands._migrate.data_classes import AssetCentricMapping
 from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_default_mappings
-from cognite_toolkit._cdf_tk.commands._migrate.issues import ChartMigrationIssue, ConversionIssue, MigrationIssue
+from cognite_toolkit._cdf_tk.commands._migrate.issues import (
+    CanvasMigrationIssue,
+    ChartMigrationIssue,
+    ConversionIssue,
+    MigrationIssue,
+)
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import AssetCentricMigrationSelector
 from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
-from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
+from cognite_toolkit._cdf_tk.exceptions import ToolkitMigrationError, ToolkitValueError
 from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.storageio._base import T_Selector
-from cognite_toolkit._cdf_tk.storageio.selectors import ChartSelector
+from cognite_toolkit._cdf_tk.storageio.selectors import CanvasSelector, ChartSelector
 from cognite_toolkit._cdf_tk.utils import humanize_collection
 from cognite_toolkit._cdf_tk.utils.useful_types import (
     T_AssetCentricResourceExtended,
@@ -245,3 +258,130 @@ class ChartMapper(DataMapper[ChartSelector, Chart, ChartWrite]):
             new_source_item = ChartSource(id=cast(str, core_ts_item.id), type=cast(str, core_ts_item.type))
             updated_source_collection.append(new_source_item)
         return updated_source_collection
+
+
+class CanvasMapper(DataMapper[CanvasSelector, IndustrialCanvas, IndustrialCanvasApply]):
+    # Note sequences are not supported in Canvas, so we do not include them here.
+    asset_centric_resource_types = tuple(("asset", "event", "timeseries", "file"))
+    DEFAULT_ASSET_VIEW = ViewId("cdf_cdm", "CogniteAsset", "v1")
+    DEFAULT_EVENT_VIEW = ViewId("cdf_cdm", "CogniteActivity", "v1")
+    DEFAULT_FILE_VIEW = ViewId("cdf_cdm", "CogniteFile", "v1")
+    DEFAULT_TIMESERIES_VIEW = ViewId("cdf_cdm", "CogniteTimeSeries", "v1")
+
+    def __init__(self, client: ToolkitClient, dry_run: bool, skip_on_missing_ref: bool = False) -> None:
+        self.client = client
+        self.dry_run = dry_run
+        self.skip_on_missing_ref = skip_on_missing_ref
+
+    def map(self, source: Sequence[IndustrialCanvas]) -> Sequence[tuple[IndustrialCanvasApply | None, MigrationIssue]]:
+        self._populate_cache(source)
+        output: list[tuple[IndustrialCanvasApply | None, MigrationIssue]] = []
+        for item in source:
+            mapped_item, issue = self._map_single_item(item)
+            output.append((mapped_item, issue))
+        return output
+
+    @property
+    def lookup_methods(self) -> dict[str, Callable]:
+        return {
+            "asset": self.client.migration.lookup.assets,
+            "event": self.client.migration.lookup.events,
+            "timeseries": self.client.migration.lookup.time_series,
+            "file": self.client.migration.lookup.files,
+        }
+
+    def _populate_cache(self, source: Sequence[IndustrialCanvas]) -> None:
+        """Populate the internal cache with references from the source canvases.
+
+        Note that the consumption views are also cached as part of the timeseries lookup.
+        """
+        ids_by_type: dict[str, set[int]] = defaultdict(set)
+        for canvas in source:
+            for ref in canvas.container_references or []:
+                if ref.container_reference_type in self.asset_centric_resource_types:
+                    ids_by_type[ref.container_reference_type].add(ref.resource_id)
+
+        for resource_type, lookup_method in self.lookup_methods.items():
+            ids = ids_by_type.get(resource_type)
+            if ids:
+                lookup_method(list(ids))
+
+    def _get_node_id(self, resource_id: int, resource_type: str) -> NodeId | None:
+        """Look up the node ID for a given resource ID and type."""
+        try:
+            return self.lookup_methods[resource_type](resource_id)
+        except KeyError:
+            raise ToolkitValueError(f"Unsupported resource type '{resource_type}' for container reference migration.")
+
+    def _get_consumer_view_id(self, resource_id: int, resource_type: str) -> ViewId:
+        """Look up the consumer view ID for a given resource ID and type."""
+        lookup_map = {
+            "asset": (self.client.migration.lookup.assets.consumer_view, self.DEFAULT_ASSET_VIEW),
+            "event": (self.client.migration.lookup.events.consumer_view, self.DEFAULT_EVENT_VIEW),
+            "timeseries": (self.client.migration.lookup.time_series.consumer_view, self.DEFAULT_TIMESERIES_VIEW),
+            "file": (self.client.migration.lookup.files.consumer_view, self.DEFAULT_FILE_VIEW),
+        }
+        if lookup_tuple := lookup_map.get(resource_type):
+            method, fallback = lookup_tuple
+            return method(resource_id) or fallback
+
+        raise ToolkitValueError(f"Unsupported resource type '{resource_type}' for container reference migration.")
+
+    def _map_single_item(self, canvas: IndustrialCanvas) -> tuple[IndustrialCanvasApply | None, CanvasMigrationIssue]:
+        update = canvas.as_write()
+        issue = CanvasMigrationIssue(canvas_external_id=canvas.canvas.external_id, canvas_name=canvas.canvas.name)
+
+        remaining_container_references: list[ContainerReferenceApply] = []
+        new_fdm_references: list[FdmInstanceContainerReferenceApply] = []
+        for ref in update.container_references or []:
+            if ref.container_reference_type not in self.asset_centric_resource_types:
+                remaining_container_references.append(ref)
+                continue
+            node_id = self._get_node_id(ref.resource_id, ref.container_reference_type)
+            if node_id is None:
+                issue.missing_reference_ids.append(ref.as_asset_centric_id())
+            else:
+                consumer_view = self._get_consumer_view_id(ref.resource_id, ref.container_reference_type)
+                fdm_ref = self.migrate_container_reference(ref, canvas.canvas.external_id, node_id, consumer_view)
+                new_fdm_references.append(fdm_ref)
+        if issue.missing_reference_ids and self.skip_on_missing_ref:
+            return None, issue
+
+        update.container_references = remaining_container_references
+        update.fdm_instance_container_references.extend(new_fdm_references)
+        if not self.dry_run:
+            backup = canvas.as_write().create_backup()
+            try:
+                self.client.canvas.industrial.create(backup)
+            except CogniteException as e:
+                raise ToolkitMigrationError(
+                    f"Failed to create backup for canvas '{canvas.canvas.name}': {e!s}. "
+                ) from e
+
+        return update, issue
+
+    @classmethod
+    def migrate_container_reference(
+        cls, reference: ContainerReferenceApply, canvas_external_id: str, instance_id: NodeId, consumer_view: ViewId
+    ) -> FdmInstanceContainerReferenceApply:
+        """Migrate a single container reference by replacing the asset-centric ID with the data model instance ID."""
+        new_id = str(uuid4())
+        new_external_id = f"{canvas_external_id}_{new_id}"
+        return FdmInstanceContainerReferenceApply(
+            external_id=new_external_id,
+            id_=new_id,
+            container_reference_type="fdmInstance",
+            instance_space=instance_id.space,
+            instance_external_id=instance_id.external_id,
+            view_space=consumer_view.space,
+            view_external_id=consumer_view.external_id,
+            view_version=consumer_view.version,
+            label=reference.label,
+            properties_=reference.properties_,
+            x=reference.x,
+            y=reference.y,
+            width=reference.width,
+            height=reference.height,
+            max_width=reference.max_width,
+            max_height=reference.max_height,
+        )

cognite_toolkit/_cdf_tk/commands/_migrate/issues.py

@@ -51,6 +51,18 @@ class ChartMigrationIssue(MigrationIssue):
         )
 
 
+class CanvasMigrationIssue(MigrationIssue):
+    type: ClassVar[str] = "canvasMigration"
+    canvas_external_id: str
+    canvas_name: str
+    missing_reference_ids: list[AssetCentricId] = Field(default_factory=list)
+
+    @property
+    def has_issues(self) -> bool:
+        """Check if there are any issues recorded in this CanvasMigrationIssue."""
+        return bool(self.missing_reference_ids)
+
+
 class ReadIssue(MigrationIssue):
     """Represents a read issue encountered during migration."""
 

cognite_toolkit/_cdf_tk/cruds/_base_cruds.py

@@ -1,6 +1,6 @@
 import sys
 from abc import ABC, abstractmethod
-from collections.abc import Hashable, Iterable, Sequence, Set, Sized
+from collections.abc import Hashable, Iterable, Sequence, Sized
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generic, TypeVar
 
@@ -9,7 +9,7 @@ from cognite.client.utils.useful_types import SequenceNotStr
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING, EXCL_FILES
+from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING, YAML_SUFFIX
 from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.resource_classes import ToolkitResource
 from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning
@@ -39,11 +39,9 @@ class Loader(ABC):
         exclude_filetypes: A set of filetypes that should be excluded from the supported filetypes.
     """
 
-    filetypes: frozenset[str]
     folder_name: str
     kind: str
     dependencies: "frozenset[type[ResourceCRUD]]" = frozenset()
-    exclude_filetypes: frozenset[str] = frozenset()
     _doc_base_url: str = "https://api-docs.cognite.com/20230101/tag/"
     _doc_url: str = ""
     sub_folder_name: str | None = None
@@ -74,13 +72,12 @@ class Loader(ABC):
     def doc_url(cls) -> str:
         return cls._doc_base_url + cls._doc_url
 
-    def find_files(self, dir_or_file: Path | None = None, include_formats: Set[str] | None = None) -> list[Path]:
+    def find_files(self, dir_or_file: Path | None = None) -> list[Path]:
         """Find all files that are supported by this loader in the given directory or file.
 
         Args:
             dir_or_file (Path): The directory or file to search in. If no path is given,
                 the build directory is used.
-            include_formats (set[str]): A set of file formats to include. If not set, all formats are included.
 
         Returns:
             list[Path]: A sorted list of all files that are supported by this loader.
@@ -94,12 +91,7 @@ class Loader(ABC):
                 raise ValueError("Invalid file type")
             return [dir_or_file]
         elif dir_or_file.is_dir():
-            file_paths = [
-                file
-                for file in dir_or_file.glob("**/*")
-                if self.is_supported_file(file) and (include_formats is None or file.suffix in include_formats)
-            ]
-            return sorted(file_paths)
+            return sorted([file for file in dir_or_file.rglob("*") if self.is_supported_file(file)])
         else:
             return []
 
@@ -118,11 +110,7 @@ class Loader(ABC):
             bool: True if the file is supported, False otherwise.
 
         """
-        if cls.filetypes and file.suffix[1:] not in cls.filetypes:
-            return False
-        if cls.exclude_filetypes and file.suffix[1:] in cls.exclude_filetypes:
-            return False
-        return file.stem.casefold().endswith(cls.kind.casefold())
+        return file.suffix in YAML_SUFFIX and file.stem.casefold().endswith(cls.kind.casefold())
 
 
 T_Loader = TypeVar("T_Loader", bound=Loader)
@@ -144,8 +132,6 @@ class ResourceCRUD(Loader, ABC, Generic[T_ID, T_ResourceRequest, T_ResourceRespo
         yaml_cls: The File format for this resource. This is used to validate the user input.
         support_drop: Whether the resource supports the drop flag.
         support_update: Whether the resource supports the update operation.
-        filetypes: The filetypes that are supported by this crud. This should not be set in the subclass, it
-            should always be yaml and yml.
         dependencies: A set of other resource cruds that must be loaded before this crud.
         parent_resource: A set of other resource cruds that are parent resources to this resource. This is used
             to determine if the iterate method should return any resources when filtering by parent ids.
@@ -158,7 +144,6 @@ class ResourceCRUD(Loader, ABC, Generic[T_ID, T_ResourceRequest, T_ResourceRespo
     # Optional to set in the subclass
     support_drop = True
     support_update = True
-    filetypes = frozenset({"yaml", "yml"})
     dependencies: "frozenset[type[ResourceCRUD]]" = frozenset()
     # For example, TransformationNotification and Schedule has Transformation as the parent resource
     # This is used in the iterate method to ensure that nothing is returned if
@@ -437,10 +422,3 @@ class DataCRUD(Loader, ABC):
     @abstractmethod
     def upload(self, state: "BuildEnvironment", dry_run: bool) -> Iterable[tuple[str, int]]:
         raise NotImplementedError
-
-    def _find_data_files(self, directory: Path) -> list[Path]:
-        return [
-            path
-            for path in directory.rglob("*")
-            if path.is_file() and path.name not in EXCL_FILES and self.is_supported_file(path)
-        ]

cognite_toolkit/_cdf_tk/cruds/_data_cruds.py

@@ -28,7 +28,6 @@ class DatapointsCRUD(DataCRUD):
     item_name = "datapoints"
     folder_name = "timeseries"
     kind = "Datapoints"
-    filetypes = frozenset({"csv", "parquet"})
     dependencies = frozenset({TimeSeriesCRUD})
     _doc_url = "Time-series/operation/postMultiTimeSeriesDatapoints"
 
@@ -43,7 +42,9 @@ class DatapointsCRUD(DataCRUD):
         resource_directories = state.built_resources[self.folder_name].get_resource_directories(self.folder_name)
 
         for resource_dir in resource_directories:
-            for datafile in self._find_data_files(resource_dir):
+            for datafile in resource_dir.rglob("*"):
+                if not datafile.stem.casefold().endswith(self.kind.casefold()):
+                    continue
                 if datafile.suffix == ".csv":
                     # The replacement is used to ensure that we read exactly the same file on Windows and Linux
                     file_content = datafile.read_bytes().replace(b"\r\n", b"\n").decode("utf-8")
@@ -52,7 +53,7 @@ class DatapointsCRUD(DataCRUD):
                 elif datafile.suffix == ".parquet":
                     data = pd.read_parquet(datafile, engine="pyarrow")
                 else:
-                    raise ValueError(f"Unsupported file type {datafile.suffix} for {datafile.name}")
+                    continue
                 timeseries_ids = list(data.columns)
                 if len(timeseries_ids) == 1:
                     ts_str = timeseries_ids[0]
@@ -87,8 +88,6 @@ class FileCRUD(DataCRUD):
     item_name = "file contents"
     folder_name = "files"
     kind = "File"
-    filetypes = frozenset()
-    exclude_filetype: frozenset[str] = frozenset({})
     dependencies = frozenset({FileMetadataCRUD, CogniteFileCRUD})
     _doc_url = "Files/operation/initFileUpload"
 
@@ -149,7 +148,6 @@ class FileCRUD(DataCRUD):
 class RawFileCRUD(DataCRUD):
     item_name = "rows"
     folder_name = "raw"
-    filetypes = frozenset({"csv", "parquet"})
     kind = "Raw"
     dependencies = frozenset({RawTableCRUD})
     _doc_url = "Raw/operation/postRows"
@@ -169,7 +167,7 @@ class RawFileCRUD(DataCRUD):
             datafile = next(
                 (
                     resource.source.path.with_suffix(f".{file_type}")
-                    for file_type in self.filetypes
+                    for file_type in ["csv", "parquet"]
                     if (resource.source.path.with_suffix(f".{file_type}").exists())
                 ),
                 None,

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py

@@ -29,20 +29,23 @@ from cognite_toolkit._cdf_tk.client.data_classes.sequences import (
     ToolkitSequenceRowsList,
     ToolkitSequenceRowsWrite,
 )
+from cognite_toolkit._cdf_tk.constants import TABLE_FORMATS, YAML_SUFFIX
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.resource_classes import AssetYAML, EventYAML, SequenceRowYAML, SequenceYAML
-from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning
+from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning, ToolkitDeprecationWarning
 from cognite_toolkit._cdf_tk.utils import load_yaml_inject_variables
 from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_hashable, diff_list_identifiable
 from cognite_toolkit._cdf_tk.utils.file import read_csv
 
 from .data_organization import DataSetsCRUD, LabelCRUD
 
+_DEPRECATION_WARNING_ISSUED = False
+
 
 @final
 class AssetCRUD(ResourceCRUD[str, AssetWrite, Asset]):
     folder_name = "classic"
-    filetypes = frozenset({"yaml", "yml", "csv", "parquet"})
     resource_cls = Asset
     resource_write_cls = AssetWrite
     yaml_cls = AssetYAML
@@ -50,6 +53,26 @@ class AssetCRUD(ResourceCRUD[str, AssetWrite, Asset]):
     dependencies = frozenset({DataSetsCRUD, LabelCRUD})
     _doc_url = "Assets/operation/createAssets"
 
+    @classmethod
+    def is_supported_file(cls, file: Path) -> bool:
+        if Flags.v08.is_enabled():
+            return super().is_supported_file(file)
+        global _DEPRECATION_WARNING_ISSUED
+        if not file.stem.casefold().endswith(cls.kind.casefold()):
+            return False
+        if file.suffix in YAML_SUFFIX:
+            return True
+        if file.suffix in TABLE_FORMATS:
+            if not _DEPRECATION_WARNING_ISSUED:
+                ToolkitDeprecationWarning(
+                    feature="deployment of asset from CSV or Parquet files",
+                    alternative="data plugin and cdf data upload commands",
+                    removal_version="0.8",
+                ).print_warning()
+                _DEPRECATION_WARNING_ISSUED = True
+            return True
+        return False
+
     @property
     def display_name(self) -> str:
         return "assets"
@@ -479,7 +502,6 @@ class SequenceRowCRUD(ResourceCRUD[str, ToolkitSequenceRowsWrite, ToolkitSequenc
 @final
 class EventCRUD(ResourceCRUD[str, EventWrite, Event]):
     folder_name = "classic"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = Event
     resource_write_cls = EventWrite
     yaml_cls = EventYAML

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py

@@ -40,7 +40,6 @@ from .group_scoped import GroupResourceScopedCRUD
 @final
 class InfieldV1CRUD(ResourceCRUD[str, APMConfigWrite, APMConfig]):
     folder_name = "cdf_applications"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = APMConfig
     resource_write_cls = APMConfigWrite
     kind = "InfieldV1"
@@ -249,7 +248,6 @@ class InfieldV1CRUD(ResourceCRUD[str, APMConfigWrite, APMConfig]):
 @final
 class InFieldLocationConfigCRUD(ResourceCRUD[NodeIdentifier, InfieldLocationConfig, InfieldLocationConfig]):
     folder_name = "cdf_applications"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = InfieldLocationConfig
     resource_write_cls = InfieldLocationConfig
     kind = "InFieldLocationConfig"
@@ -349,7 +347,6 @@ class InFieldLocationConfigCRUD(ResourceCRUD[NodeIdentifier, InfieldLocationConf
 @final
 class InFieldCDMLocationConfigCRUD(ResourceCRUD[NodeIdentifier, InFieldCDMLocationConfig, InFieldCDMLocationConfig]):
     folder_name = "cdf_applications"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = InFieldCDMLocationConfig
     resource_write_cls = InFieldCDMLocationConfig
     kind = "InFieldCDMLocationConfig"

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/migration.py

@@ -21,7 +21,6 @@ from .datamodel import SpaceCRUD, ViewCRUD
 @final
 class ResourceViewMappingCRUD(ResourceCRUD[str, ResourceViewMappingApply, ResourceViewMapping]):
     folder_name = "migration"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = ResourceViewMapping
     resource_write_cls = ResourceViewMappingApply
     kind = "ResourceViewMapping"

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/relationship.py

@@ -23,7 +23,6 @@ from .timeseries import TimeSeriesCRUD
 @final
 class RelationshipCRUD(ResourceCRUD[str, RelationshipWrite, Relationship]):
     folder_name = "classic"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = Relationship
     resource_write_cls = RelationshipWrite
     kind = "Relationship"

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py

@@ -19,7 +19,6 @@ from .datamodel import ContainerCRUD
 @final
 class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse]):
     folder_name = "streams"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = StreamResponse
     resource_write_cls = StreamRequest
     kind = "Streams"

cognite_toolkit/_cdf_tk/cruds/_worker.py

@@ -14,6 +14,7 @@ from yaml import YAMLError
 
 from cognite_toolkit._cdf_tk.constants import TABLE_FORMATS
 from cognite_toolkit._cdf_tk.exceptions import ToolkitWrongResourceError, ToolkitYAMLFormatError
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarning, catch_warnings
 from cognite_toolkit._cdf_tk.utils import to_diff
 
@@ -47,12 +48,16 @@ class ResourceWorker(Generic[T_ID, T_ResourceRequest, T_ResourceResponse]):
     ) -> list[Path]:
         filepaths = self.loader.find_files(directory)
 
-        for read_module in read_modules or []:
-            if resource_dir := read_module.resource_dir_path(self.loader.folder_name):
-                # As of 05/11/24, Asset support csv and parquet files in addition to YAML.
-                # These table formats are not built, i.e., no variable replacement is done,
-                # so we load them directly from the source module.
-                filepaths.extend(self.loader.find_files(resource_dir, include_formats=TABLE_FORMATS))
+        if not Flags.v08.is_enabled():
+            for read_module in read_modules or []:
+                if resource_dir := read_module.resource_dir_path(self.loader.folder_name):
+                    # As of 05/11/24, Asset support csv and parquet files in addition to YAML.
+                    # These table formats are not built, i.e., no variable replacement is done,
+                    # so we load them directly from the source module.
+                    table_files = [
+                        file for file in self.loader.find_files(resource_dir) if file.suffix in TABLE_FORMATS
+                    ]
+                    filepaths.extend(table_files)
 
         if not sort:
             return filepaths

cognite_toolkit/_cdf_tk/storageio/_applications.py

@@ -1,6 +1,7 @@
 from collections.abc import Iterable, Sequence
 from typing import Any
 
+from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.canvas import (
     IndustrialCanvas,
     IndustrialCanvasApply,
@@ -127,6 +128,15 @@ class ChartIO(UploadableStorageIO[ChartSelector, Chart, ChartWrite]):
 
 
 class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialCanvasApply]):
+    """Download and upload Industrial Canvases to/from CDF.
+
+    Args:
+        client (ToolkitClient): The Cognite Toolkit client to use for API interactions.
+        exclude_existing_version (bool): Whether to exclude the 'existingVersion' field when uploading canvases.
+            Defaults to True. If you set this to False, the upload may fail if the existing version in CDF is
+            lower or equal to the one in the uploaded data.
+    """
+
     KIND = "IndustrialCanvas"
     SUPPORTED_DOWNLOAD_FORMATS = frozenset({".ndjson"})
     SUPPORTED_COMPRESSIONS = frozenset({".gz"})
@@ -134,6 +144,10 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
     CHUNK_SIZE = 10
     BASE_SELECTOR = CanvasSelector
 
+    def __init__(self, client: ToolkitClient, exclude_existing_version: bool = True) -> None:
+        super().__init__(client)
+        self.exclude_existing_version = exclude_existing_version
+
     def as_id(self, item: IndustrialCanvas) -> str:
         return item.as_id()
 
@@ -171,12 +185,19 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
         results: list[HTTPMessage] = []
         for item in data_chunk:
             instances = item.item.as_instances()
+            items: list[dict[str, JsonVal]] = []
+            for instance in instances:
+                dumped = instance.dump()
+                if self.exclude_existing_version:
+                    dumped.pop("existingVersion", None)
+                items.append(dumped)
+
             responses = http_client.request_with_retries(
                 message=SimpleBodyRequest(
                     endpoint_url=config.create_api_url("/models/instances"),
                     method="POST",
                     # MyPy does not understand that .dump is valid json
-                    body_content={"items": [instance.dump() for instance in instances]},
+                    body_content={"items": items},  # type: ignore[dict-item]
                 )
             )
             results.extend(responses.as_item_responses(item.source_id))

cognite_toolkit/_cdf_tk/tk_warnings/other.py

@@ -136,15 +136,19 @@ class MissingCapabilityWarning(GeneralWarning):
 
 @dataclass(frozen=True)
 class ToolkitDeprecationWarning(ToolkitWarning, DeprecationWarning):
+    severity = SeverityLevel.HIGH
     message: ClassVar[str] = "The '{feature}' is deprecated and will be removed in a future version."
 
     feature: str
     alternative: str | None = None
+    removal_version: str | None = None
 
     def get_message(self) -> str:
         msg = self.message.format(feature=self.feature)
         if self.alternative:
             msg += f"\nUse {self.alternative!r} instead."
+        if self.removal_version:
+            msg += f"\nIt will be removed in version {self.removal_version}."
 
         return msg
 

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.7.25
+      image: cognite/toolkit:0.7.27
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.7.25
+      image: cognite/toolkit:0.7.27
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.25"
+version = "0.7.27"
 
 
 [plugins]

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.7.25"
+__version__ = "0.7.27"

{cognite_toolkit-0.7.25.dist-info → cognite_toolkit-0.7.27.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.25
+Version: 0.7.27
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>

{cognite_toolkit-0.7.25.dist-info → cognite_toolkit-0.7.27.dist-info}/RECORD

@@ -9,7 +9,7 @@ cognite_toolkit/_cdf_tk/apps/_dev_app.py,sha256=FaY67PFdKwdiMKgJbTcjHT1X2Xfbog2P
 cognite_toolkit/_cdf_tk/apps/_download_app.py,sha256=2nPn9P_9br9poynSpKKSZF7WYTYT--BfxlxXkSEeH-8,41156
 cognite_toolkit/_cdf_tk/apps/_dump_app.py,sha256=EPq6fWSaScj9ncKfRY253rRJ37er47PIM60IFgkQK_k,37127
 cognite_toolkit/_cdf_tk/apps/_landing_app.py,sha256=YR9z83OY7PhhgBVC5gmRLgo9iTXoGoZfRhOU3gd_r2o,888
-cognite_toolkit/_cdf_tk/apps/_migrate_app.py,sha256=g4S_53kbIgk57ziPLdRMuR6xUe434gkMqa69VmVm5Vg,39619
+cognite_toolkit/_cdf_tk/apps/_migrate_app.py,sha256=_woM0D2j6VzuYC0LJKteALbQ4U8vGj0B1LSBj_WszKQ,41198
 cognite_toolkit/_cdf_tk/apps/_modules_app.py,sha256=t0SPvulgbgkF_OO2E68mQ_ZUcJ6HoaurYe0IkmXie0o,7449
 cognite_toolkit/_cdf_tk/apps/_profile_app.py,sha256=vSRJW54bEvIul8_4rOqyOYA7ztXx7TFOvZRZWZTxMbg,7007
 cognite_toolkit/_cdf_tk/apps/_purge.py,sha256=KYI1wFy7yHFEM1qJnTYc4_8E2FVGu4QhPsWsxop1sZA,14242
@@ -93,21 +93,20 @@ cognite_toolkit/_cdf_tk/client/testing.py,sha256=mXqEXPMZcbETrXBn6D-SiAcjD7xAkuu
 cognite_toolkit/_cdf_tk/client/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/client/utils/_concurrency.py,sha256=3GtQbKDaosyKHEt-KzxKK9Yie4TvZPdoou2vUk6dUa8,2298
 cognite_toolkit/_cdf_tk/client/utils/_http_client.py,sha256=oXNKrIaizG4WiSAhL_kSCHAuL4aaaEhCU4pOJGxh6Xs,483
-cognite_toolkit/_cdf_tk/commands/__init__.py,sha256=HQIHw18EU09fdo7bxbDXi8-0Lc5t2KE6BpuhNZehpqo,1402
+cognite_toolkit/_cdf_tk/commands/__init__.py,sha256=wXRbOwMyhGjMj83_bXTOUXtCh70YS4-eiYzzCMN_YOs,1390
 cognite_toolkit/_cdf_tk/commands/_base.py,sha256=1gl8Y-yqfedRMfdbwM3iPTIUIZriX1UvC1deLsJSJwM,2667
 cognite_toolkit/_cdf_tk/commands/_changes.py,sha256=sU0KaTtPVSJgAZcaZ1Tkcajj36pmhd13kh7V8QbIED8,22987
 cognite_toolkit/_cdf_tk/commands/_cli_commands.py,sha256=TK6U_rm6VZT_V941kTyHMoulWgJzbDC8YIIQDPJ5x3w,1011
 cognite_toolkit/_cdf_tk/commands/_download.py,sha256=dVddH9t7oGx1kdQ3CCYYQb96Uxxy-xC8Opph98lo46U,6869
-cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py,sha256=i5ldcTah59K0E4fH5gHTV0GRvtDCEvVses9WQzn9Lno,226
-cognite_toolkit/_cdf_tk/commands/_migrate/canvas.py,sha256=R-z0yfOFcJZj-zRLhN-7z_-SLxqzSmONMgrbzNF9dGs,8843
+cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py,sha256=8ki04tJGH1dHdF2NtVF4HyhaC0XDDS7onrH_nvd9KtE,153
 cognite_toolkit/_cdf_tk/commands/_migrate/command.py,sha256=l2P0Em05aEJvNZH4WkEIm-QfO3TAjG1rc_YxELuQIQM,14214
 cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py,sha256=Ew9JRYrd-Ol9G9csTzpnhXAgCFnX67MwDYOTsdJLP3E,16803
 cognite_toolkit/_cdf_tk/commands/_migrate/creators.py,sha256=FTu7w3G8KyPY8pagG3KdPpOmpLcjehaAg2auEy6iM7A,9605
 cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py,sha256=_vMS_qAPj4yup1VnmmojPVigAZtyPQH7PM0Raby5tao,10619
-cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py,sha256=nNaM8D1-kKfgkah28L7S8agA-DRcpeeEd2kazyhodIc,11713
+cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py,sha256=vixRnB-4sjDCE2DafMJs_oApVe5HgwOrzSPt49ypUgM,18491
 cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py,sha256=i1eUsNX6Dueol9STIEwyksBnBsWUk13O8qHIjW964pM,7860
 cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py,sha256=ERn3qFrJFXdtXaMjHq3Gk7MxH03MGFk3FrtWCOBJQts,5544
-cognite_toolkit/_cdf_tk/commands/_migrate/issues.py,sha256=L2-kODPavEwcuhte7EXANK2-rH7reiq-uNqr-3ub-no,6575
+cognite_toolkit/_cdf_tk/commands/_migrate/issues.py,sha256=n8en744-r7GL9eUyxEojFes1yk69V04SnlpVXHrdPOQ,6972
 cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py,sha256=wrdBH5P6NgiZQSYLR0iJ3ZvqfQ5fY-_Ne2yKv9E1g4o,16277
 cognite_toolkit/_cdf_tk/commands/_migrate/prepare.py,sha256=RfqaNoso5CyBwc-p6ckwcYqBfZXKhdJgdGIyd0TATaI,2635
 cognite_toolkit/_cdf_tk/commands/_migrate/selectors.py,sha256=N1H_-rBpPUD6pbrlcofn1uEK1bA694EUXEe1zIXeqyo,2489
@@ -132,33 +131,33 @@ cognite_toolkit/_cdf_tk/commands/resources.py,sha256=NeHVA1b1TMsP-2wgd5u1vif_N6n
 cognite_toolkit/_cdf_tk/commands/run.py,sha256=JyX9jLEQej9eRrHVCCNlw4GuF80qETSol3-T5CCofgw,37331
 cognite_toolkit/_cdf_tk/constants.py,sha256=TplKm2J9pGRHq7nAnLI0caTMHetS04OIz3hfq-jvGzo,7236
 cognite_toolkit/_cdf_tk/cruds/__init__.py,sha256=noZQvCaAe6u67NSuJahil5Q44_tk5aGFWKyi9wBot1U,6675
-cognite_toolkit/_cdf_tk/cruds/_base_cruds.py,sha256=w8Qt00_AF1cM47ttBD96CTi9h5iM7QHFpGEEPKZGnnI,18663
-cognite_toolkit/_cdf_tk/cruds/_data_cruds.py,sha256=4inL7ACmIQs3i9rVOqKG4uOnp3Ywhwg_Fmrm_9hWpyI,8890
+cognite_toolkit/_cdf_tk/cruds/_base_cruds.py,sha256=6I1P0kZKxX7CDHM3_xPNnMJQi_QEviE_XnhpM_PAULc,17652
+cognite_toolkit/_cdf_tk/cruds/_data_cruds.py,sha256=ggV8Hao5CzJvWkbVEm1u3DgSL7yQwxfMlsTe2TKLnYA,8746
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/__init__.py,sha256=diUM4kSbOw3YfdkmByy20BgzJcEM5ylYrnnEIoQEddk,2945
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/agent.py,sha256=2UjX0m85fw_zt4EpCm5Ihz2gE1AlgOgR8-7Pr8M2c4g,5128
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py,sha256=mswG-Fp_nfgociGTJ_aIG18we2nFNurcyObPxD9WKlA,24540
-cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py,sha256=r9yY02GS3VEDgf85DCBrt06t4kM0o6uGqUmQm_NdQuw,25661
+cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py,sha256=k1PPjCq8XyuD2SZoU40ffMenboivO2sSJvYvbZljQzo,26562
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/configuration.py,sha256=plVGY-hvT01XC96C6hrabL07NaLphNWUq2iNI3m2Oyw,5811
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/data_organization.py,sha256=U0ItuoNr1KEtoFQAiIe-K19_72ht9-kGndFVgF-iC10,9524
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py,sha256=SagiSp3JERgEU3SnkjQ76vrxSM7gRA17lvoh0BW4KeQ,64867
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/extraction_pipeline.py,sha256=a2HywkruYNJGLZxqOjlp8mrpRGtJDPqIb6qY00eUbEI,17701
-cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py,sha256=Tzrkfp9mtseUgnMdpVy_iBTup8S_1caypR5tknedeH4,20791
+cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py,sha256=dcC850Vyvc5Hfi9Z3MfXE8s_q14Hqq4EqegFz_V6aCI,20662
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/file.py,sha256=vyeRsiIOEbUeYslBsgXoyCk5hozDsubUilA7bdjqS5c,14855
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/function.py,sha256=v3kjn3igwTF53LJ6pp0O8d4S1XFJ1eXQGCchWAcaAx0,28439
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/group_scoped.py,sha256=WEg8-CxMP64WfE_XXIlH114zM51K0uLaYa4atd992zI,1690
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/hosted_extractors.py,sha256=P0hlXK0_FmO86U-gDHMHz8N0vpDtPoKupiQfhNP5KLE,14619
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/industrial_tool.py,sha256=QrgSCcLN0NtpQuP7zcCUYaWoiq3JiUB2j0A15d8MNNc,7856
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/location.py,sha256=-FDfZ-iPsWh_65HCze8M3-5Ijo3sr_n-AT-6FENSjpk,12076
-cognite_toolkit/_cdf_tk/cruds/_resource_cruds/migration.py,sha256=v_aQbZwUZXBAj7AjK22f-7mbcKNdB5FBlyTQiLaCPAw,4499
+cognite_toolkit/_cdf_tk/cruds/_resource_cruds/migration.py,sha256=rlgi9_4c0Yyhx6m22He3bOenpw46nO-zBxoMrpjcHVs,4456
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/raw.py,sha256=rKTmIYD2_nm7LeIBa10iMZreNvw00OuQu-wC0gyiGQg,12072
-cognite_toolkit/_cdf_tk/cruds/_resource_cruds/relationship.py,sha256=G_pUKvIpdF54SqDQsCLYcWGM6twZJ_JA-xeFAYCgtdk,6194
+cognite_toolkit/_cdf_tk/cruds/_resource_cruds/relationship.py,sha256=90HVz6KR_hOIsbOBFbt2oORWJ2lWsaQ7TAiPtFmzTi0,6151
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/robotics.py,sha256=IeDks5yJFta8Tp4SmikWjVxn0kyeCv44Pf3Jw9y5zQc,16074
-cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py,sha256=N-6YgllJbv7XudzkOAUpPktLsnFlCFU6fzabV9oX3N0,3097
+cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py,sha256=1GAIn6G3ZC11ZAgRW-HKaJltMhIqSKH6_ttdEMGgFN0,3054
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/three_d_model.py,sha256=5jIRF2JqosVHyAi2Ek6H38K2FWcNqrbPOR6MTSIEAQI,7320
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/timeseries.py,sha256=rMkA78K9CZqUu7YBDiLgnaZgPcIsWdT-J3nB4xwCJWw,23032
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py,sha256=ndhoGIC6tBjNmwI0wDfg8v_6dJfCnHUnTienYOtKwC8,33876
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/workflow.py,sha256=dVkf7OpZHV2H7V3zRU4BzUqDxgAudiWBJPvGeydATCg,26035
-cognite_toolkit/_cdf_tk/cruds/_worker.py,sha256=3UABlgb_LhM4dXpuruuxsVzvJuKvCPqgEugNuqSvqJQ,9054
+cognite_toolkit/_cdf_tk/cruds/_worker.py,sha256=QeZjziBCePilyI4WfTZvLfSHGT3qJGCMhugsoO1SVtU,9285
 cognite_toolkit/_cdf_tk/data_classes/__init__.py,sha256=4zL-zR3lgQTCWfcy28LK0HEcukQOndPEFXVqfYfdKHU,1720
 cognite_toolkit/_cdf_tk/data_classes/_base.py,sha256=0jy9VrYIO6iRgOZIcRASv-xIQjU3QbMICffEEIqzx6A,2673
 cognite_toolkit/_cdf_tk/data_classes/_build_files.py,sha256=ARZpzcpmcbtG5Jg391d7A-7MJCFeUqqiDfJlO14cvvI,953
@@ -240,7 +239,7 @@ cognite_toolkit/_cdf_tk/resource_classes/workflow_trigger.py,sha256=aSN0WFPupQ38
 cognite_toolkit/_cdf_tk/resource_classes/workflow_version.py,sha256=ui724EaM9Nlm3wTnm7Givgv6GLQ-xbsfZgidyRKv09U,2991
 cognite_toolkit/_cdf_tk/storageio/__init__.py,sha256=h5Wr4i7zNIgsslrsRJxmp7ls4bNRKl0uZzQ7GLRMP7g,1920
 cognite_toolkit/_cdf_tk/storageio/_annotations.py,sha256=JI_g18_Y9S7pbc9gm6dZMyo3Z-bCndJXF9C2lOva0bQ,4848
-cognite_toolkit/_cdf_tk/storageio/_applications.py,sha256=ozWeuTqay1_GSFuQZUxXJspsBhLEBIS7EksAi93B8_4,18722
+cognite_toolkit/_cdf_tk/storageio/_applications.py,sha256=M7FEK4xC0BjP2i6FyYs1589zEA3afJiOKCzY56RV6NU,19685
 cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=TirKLSNPoLqKjczsw0djWAsR0VvopwmU23aUxrBOJN8,32464
 cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=ElvqhIEBnhcz0yY1Ds164wVN0_7CFNK-uT0-z7LcR9U,13067
 cognite_toolkit/_cdf_tk/storageio/_data_classes.py,sha256=s3TH04BJ1q7rXndRhEbVMEnoOXjxrGg4n-w9Z5uUL-o,3480
@@ -260,7 +259,7 @@ cognite_toolkit/_cdf_tk/storageio/selectors/_raw.py,sha256=sZq9C4G9DMe3S46_usKet
 cognite_toolkit/_cdf_tk/tk_warnings/__init__.py,sha256=U9bT-G2xKrX6mmtZ7nZ1FfQeCjNKfKP_p7pev90dwOE,2316
 cognite_toolkit/_cdf_tk/tk_warnings/base.py,sha256=cX8TCmb56gqx3lc7dankXuqpm5HGASJ4wTb07-MCJWs,4401
 cognite_toolkit/_cdf_tk/tk_warnings/fileread.py,sha256=d2Kx6YyLmCkyFNjK8MO6eKGceCIEaFLZ4LYcG-EjnuM,8947
-cognite_toolkit/_cdf_tk/tk_warnings/other.py,sha256=4T-WRqMG-KKFS0AV3w6ilea22G_MTXFcOKpbd2z6MNE,5624
+cognite_toolkit/_cdf_tk/tk_warnings/other.py,sha256=D8EubXyW4qigscBEiedQJuT5c6yYoFIEhBPa1DggD3I,5808
 cognite_toolkit/_cdf_tk/tracker.py,sha256=jhxzI8LOSZw3zDBPsTLW3zC2YcQK2abp_aVtRKcUIwE,5913
 cognite_toolkit/_cdf_tk/utils/__init__.py,sha256=-X01eYNwz3l0W2jby0DZzlDIe9HEhUUj-dK8DBhYki8,1413
 cognite_toolkit/_cdf_tk/utils/_auxiliary.py,sha256=tvvgFiWwLOCVDkPg83U5XqBLfOOt_gI3697EQr7-GSE,1198
@@ -303,14 +302,14 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=AE3C_roD1yRh00t5w3C6W7OLAeQhTGAXMalj68JcV2o,667
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=5ivDvAj_lDr5njBvUERlTKQ2JBUzm0mA9ADWcnIPdPo,2430
-cognite_toolkit/_resources/cdf.toml,sha256=mTx8mZuU7-yKdyAz_qHHImkMLrl0UFv8BtQ0Vjnjyps,475
-cognite_toolkit/_version.py,sha256=iKBQaXokAaY4fPUZd7GsPmMNh78mp8Ix8lWv4BxZua4,23
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=zf9hdCJ3SerGDrnmBTLTn3CMH0hGGYTh-hdVh3A115M,667
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=mRRQGQOYBRdXpgyuC1O1rEH5owcwsLVI_CvEyq3iqnk,2430
+cognite_toolkit/_resources/cdf.toml,sha256=xSpuqW3POVKkqGWVtHMIiLc2dQZeGZpdqh1sJPHKC8A,475
+cognite_toolkit/_version.py,sha256=fHi3FiqQRMkjl3o8ATyV9gZrBSNTVvt5m0WxJ-NYHTA,23
 cognite_toolkit/config.dev.yaml,sha256=M33FiIKdS3XKif-9vXniQ444GTZ-bLXV8aFH86u9iUQ,332
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.7.25.dist-info/WHEEL,sha256=93kfTGt3a0Dykt_T-gsjtyS5_p8F_d6CE1NwmBOirzo,79
-cognite_toolkit-0.7.25.dist-info/entry_points.txt,sha256=EtZ17K2mUjh-AY0QNU1CPIB_aDSSOdmtNI_4Fj967mA,84
-cognite_toolkit-0.7.25.dist-info/METADATA,sha256=Xo2lngR5nSAc9EZePQYgLKJCTiWQwWfz5dUvxCD8O_I,4507
-cognite_toolkit-0.7.25.dist-info/RECORD,,
+cognite_toolkit-0.7.27.dist-info/WHEEL,sha256=93kfTGt3a0Dykt_T-gsjtyS5_p8F_d6CE1NwmBOirzo,79
+cognite_toolkit-0.7.27.dist-info/entry_points.txt,sha256=EtZ17K2mUjh-AY0QNU1CPIB_aDSSOdmtNI_4Fj967mA,84
+cognite_toolkit-0.7.27.dist-info/METADATA,sha256=OoJ1UzFNOjp3Tja-1bdSe8LrbCq167Bq0lBnRGugbc8,4507
+cognite_toolkit-0.7.27.dist-info/RECORD,,

cognite_toolkit/_cdf_tk/commands/_migrate/canvas.py

@@ -1,201 +0,0 @@
-from uuid import uuid4
-
-from cognite.client.data_classes.capabilities import (
-    Capability,
-    DataModelInstancesAcl,
-    DataModelsAcl,
-    SpaceIDScope,
-)
-from cognite.client.exceptions import CogniteException
-
-from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.canvas import (
-    CANVAS_INSTANCE_SPACE,
-    Canvas,
-    ContainerReferenceApply,
-    FdmInstanceContainerReferenceApply,
-)
-from cognite_toolkit._cdf_tk.client.data_classes.migration import InstanceSource
-from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
-from cognite_toolkit._cdf_tk.commands._migrate.data_model import (
-    INSTANCE_SOURCE_VIEW_ID,
-    MODEL_ID,
-    RESOURCE_VIEW_MAPPING_VIEW_ID,
-)
-from cognite_toolkit._cdf_tk.exceptions import AuthenticationError, ToolkitMigrationError
-from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, LowSeverityWarning, MediumSeverityWarning
-from cognite_toolkit._cdf_tk.utils import humanize_collection
-from cognite_toolkit._cdf_tk.utils.interactive_select import InteractiveCanvasSelect
-
-
-class MigrationCanvasCommand(ToolkitCommand):
-    canvas_schema_space = Canvas.get_source().space
-    # Note sequences are not supported in Canvas, so we do not include them here.
-    asset_centric_resource_types = frozenset({"asset", "event", "file", "timeseries"})
-
-    def migrate_canvas(
-        self,
-        client: ToolkitClient,
-        external_ids: list[str] | None = None,
-        dry_run: bool = False,
-        verbose: bool = False,
-    ) -> None:
-        self.validate_access(
-            client,
-            instance_spaces=[CANVAS_INSTANCE_SPACE],
-            schema_spaces=[self.canvas_schema_space, INSTANCE_SOURCE_VIEW_ID.space],
-        )
-        self.validate_migration_model_available(client)
-        external_ids = external_ids or InteractiveCanvasSelect(client).select_external_ids()
-        if external_ids is None or not external_ids:
-            self.console("No canvases selected for migration.")
-            return
-        action = "Would migrate" if dry_run else "Migrating"
-        self.console(f"{action} {len(external_ids)} canvases.")
-        for external_id in external_ids:
-            self._migrate_single_canvas(client, external_id, dry_run=dry_run, verbose=verbose)
-
-    def _migrate_single_canvas(
-        self,
-        client: ToolkitClient,
-        external_id: str,
-        dry_run: bool = False,
-        verbose: bool = False,
-    ) -> None:
-        canvas = client.canvas.industrial.retrieve(external_id=external_id)
-        if canvas is None:
-            self.warn(MediumSeverityWarning(f"Canvas with external ID '{external_id}' not found. Skipping.. "))
-            return
-        update = canvas.as_write()
-        to_migrate = [
-            ref
-            for ref in update.container_references
-            if ref.container_reference_type in self.asset_centric_resource_types
-        ]
-        if not to_migrate:
-            self.warn(
-                LowSeverityWarning(
-                    f"Canvas with name '{canvas.canvas.name}' does not have any asset-centric references. Skipping.. "
-                )
-            )
-        if verbose:
-            self.console(f"Found canvas: {canvas.canvas.name}")
-        reference_ids = [ref.as_asset_centric_id() for ref in to_migrate]
-        instance_sources = client.migration.instance_source.retrieve(reference_ids)
-        source_by_reference_id = {source.as_asset_centric_id(): source for source in instance_sources}
-        missing = set(reference_ids) - set(source_by_reference_id.keys())
-        if missing:
-            self.warn(
-                HighSeverityWarning(
-                    f"Canvas '{canvas.canvas.name}' has references to resources that are not been migrated: {humanize_collection(missing)}. Skipping.. "
-                )
-            )
-            return
-        if dry_run:
-            self.console(
-                f"Canvas '{canvas.canvas.name}' is ready for migration all {len(instance_sources)} references asset-centric resources found."
-            )
-            return
-        if verbose:
-            self.console(
-                f"Migrating canvas '{canvas.canvas.name}' with {len(instance_sources)} references to asset-centric resources."
-            )
-        backup = canvas.as_write().create_backup()
-
-        update.container_references = [
-            ref
-            for ref in update.container_references
-            if ref.container_reference_type not in self.asset_centric_resource_types
-        ]
-        for ref in to_migrate:
-            source = source_by_reference_id[ref.as_asset_centric_id()]
-            fdm_ref = self.migrate_container_reference(ref, source, canvas.canvas.external_id)
-            update.fdm_instance_container_references.append(fdm_ref)
-
-        try:
-            client.canvas.industrial.create(backup)
-        except CogniteException as e:
-            raise ToolkitMigrationError(f"Failed to create backup for canvas '{canvas.canvas.name}': {e!s}. ") from e
-        try:
-            client.canvas.industrial.update(update)
-        except CogniteException as e:
-            raise ToolkitMigrationError(
-                f"Failed to migrate canvas '{canvas.canvas.name}': {e!s}. A backup was created with external ID '{backup.canvas.external_id}'."
-            ) from e
-        else:
-            self.console(
-                f'Canvas "{canvas.canvas.name}" migrated successfully with {len(to_migrate)} references to data model instances.'
-            )
-
-    @classmethod
-    def migrate_container_reference(
-        cls, reference: ContainerReferenceApply, source: InstanceSource, canvas_external_id: str
-    ) -> FdmInstanceContainerReferenceApply:
-        """Migrate a single container reference by replacing the asset-centric ID with the data model instance ID."""
-        consumer_view = source.consumer_view()
-        new_id = str(uuid4())
-        new_external_id = f"{canvas_external_id}_{new_id}"
-        return FdmInstanceContainerReferenceApply(
-            external_id=new_external_id,
-            id_=new_id,
-            container_reference_type="fdmInstance",
-            instance_space=source.space,
-            instance_external_id=source.external_id,
-            view_space=consumer_view.space,
-            view_external_id=consumer_view.external_id,
-            view_version=consumer_view.version,
-            label=reference.label,
-            properties_=reference.properties_,
-            x=reference.x,
-            y=reference.y,
-            width=reference.width,
-            height=reference.height,
-            max_width=reference.max_width,
-            max_height=reference.max_height,
-        )
-
-    @staticmethod
-    def validate_access(
-        client: ToolkitClient,
-        instance_spaces: list[str] | None = None,
-        schema_spaces: list[str] | None = None,
-    ) -> None:
-        required_capabilities: list[Capability] = []
-        if instance_spaces is not None:
-            required_capabilities.append(
-                DataModelInstancesAcl(
-                    actions=[
-                        DataModelInstancesAcl.Action.Read,
-                        DataModelInstancesAcl.Action.Write,
-                        DataModelInstancesAcl.Action.Write_Properties,
-                    ],
-                    scope=SpaceIDScope(instance_spaces),
-                )
-            )
-        if schema_spaces is not None:
-            required_capabilities.append(
-                DataModelsAcl(actions=[DataModelsAcl.Action.Read], scope=SpaceIDScope(schema_spaces)),
-            )
-        if missing := client.iam.verify_capabilities(required_capabilities):
-            raise AuthenticationError(f"Missing required capabilities: {humanize_collection(missing)}.", missing)
-
-    @staticmethod
-    def validate_migration_model_available(client: ToolkitClient) -> None:
-        models = client.data_modeling.data_models.retrieve([MODEL_ID], inline_views=False)
-        if not models:
-            raise ToolkitMigrationError(
-                f"The migration data model {MODEL_ID!r} does not exist. "
-                "Please run the `cdf migrate prepare` command to deploy the migration data model."
-            )
-        elif len(models) > 1:
-            raise ToolkitMigrationError(
-                f"Multiple migration models {MODEL_ID!r}. "
-                "Please delete the duplicate models before proceeding with the migration."
-            )
-        model = models[0]
-        missing_views = {INSTANCE_SOURCE_VIEW_ID, RESOURCE_VIEW_MAPPING_VIEW_ID} - set(model.views or [])
-        if missing_views:
-            raise ToolkitMigrationError(
-                f"Invalid migration model. Missing views {humanize_collection(missing_views)}. "
-                f"Please run the `cdf migrate prepare` command to deploy the migration data model."
-            )