cognite-toolkit 0.7.25__tar.gz → 0.7.27__tar.gz

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.25
+Version: 0.7.27
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/cognite_toolkit/_cdf_tk/apps/_migrate_app.py

@@ -8,17 +8,14 @@ from cognite.client.data_classes import Annotation
 from cognite.client.data_classes.data_modeling import ContainerId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.commands import (
-    MigrationCanvasCommand,
-    MigrationPrepareCommand,
-)
+from cognite_toolkit._cdf_tk.commands import MigrationPrepareCommand
 from cognite_toolkit._cdf_tk.commands._migrate import MigrationCommand
 from cognite_toolkit._cdf_tk.commands._migrate.creators import (
     InfieldV2ConfigCreator,
     InstanceSpaceCreator,
     SourceSystemCreator,
 )
-from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper, ChartMapper
+from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper, CanvasMapper, ChartMapper
 from cognite_toolkit._cdf_tk.commands._migrate.migration_io import (
     AnnotationMigrationIO,
     AssetCentricMigrationIO,
@@ -28,14 +25,15 @@ from cognite_toolkit._cdf_tk.commands._migrate.selectors import (
     MigrateDataSetSelector,
     MigrationCSVFileSelector,
 )
-from cognite_toolkit._cdf_tk.storageio import ChartIO
-from cognite_toolkit._cdf_tk.storageio.selectors import ChartExternalIdSelector
+from cognite_toolkit._cdf_tk.storageio import CanvasIO, ChartIO
+from cognite_toolkit._cdf_tk.storageio.selectors import CanvasExternalIdSelector, ChartExternalIdSelector
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._cdf_tk.utils.cli_args import parse_view_str
 from cognite_toolkit._cdf_tk.utils.interactive_select import (
     AssetInteractiveSelect,
     DataModelingSelect,
     FileMetadataInteractiveSelect,
+    InteractiveCanvasSelect,
     InteractiveChartSelect,
     ResourceViewMappingInteractiveSelect,
 )
@@ -855,6 +853,23 @@ class MigrateApp(typer.Typer):
                 "performed to select the Canvas to migrate."
             ),
         ] = None,
+        skip_on_missing_ref: Annotated[
+            bool,
+            typer.Option(
+                "--skip-on-missing-ref",
+                "-s",
+                help="If set, the migration will skip Canvases that reference resources that have not been migrated to data modeling. "
+                "If not set, the migration will continue but the result will exclude the missing references.",
+            ),
+        ] = False,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where migration logs will be stored.",
+            ),
+        ] = Path(f"migration_logs_{TODAY}"),
         dry_run: Annotated[
             bool,
             typer.Option(
@@ -880,12 +895,24 @@ class MigrateApp(typer.Typer):
         is populated with the mapping from Asset-Centric resources to the new data modeling resources.
         """
         client = EnvironmentVariables.create_from_environment().get_client()
+        if external_id is None:
+            interactive = InteractiveCanvasSelect(client)
+            external_id = interactive.select_external_ids()
+            log_dir = questionary.path("Specify log directory for migration logs:", default=str(log_dir)).ask()
+            dry_run = questionary.confirm("Do you want to perform a dry run?", default=dry_run).ask()
+            verbose = questionary.confirm("Do you want verbose output?", default=verbose).ask()
+            if any(res is None for res in [log_dir, dry_run, verbose]):
+                raise typer.Abort()
+            log_dir = Path(log_dir)
 
-        cmd = MigrationCanvasCommand()
+        cmd = MigrationCommand()
+        selector = CanvasExternalIdSelector(external_ids=tuple(external_id))
         cmd.run(
-            lambda: cmd.migrate_canvas(
-                client,
-                external_ids=external_id,
+            lambda: cmd.migrate(
+                selected=selector,
+                data=CanvasIO(client, exclude_existing_version=True),
+                mapper=CanvasMapper(client, dry_run=dry_run, skip_on_missing_ref=skip_on_missing_ref),
+                log_dir=log_dir,
                 dry_run=dry_run,
                 verbose=verbose,
             )

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/cognite_toolkit/_cdf_tk/commands/__init__.py

@@ -1,6 +1,6 @@
 from ._download import DownloadCommand
 from ._migrate import (
-    MigrationCanvasCommand,
+    MigrationCommand,
     MigrationPrepareCommand,
 )
 from ._profile import ProfileAssetCentricCommand, ProfileAssetCommand, ProfileRawCommand, ProfileTransformationCommand
@@ -30,7 +30,7 @@ __all__ = [
     "DownloadCommand",
     "DumpResourceCommand",
     "InitCommand",
-    "MigrationCanvasCommand",
+    "MigrationCommand",
     "MigrationPrepareCommand",
     "ModulesCommand",
     "ProfileAssetCentricCommand",

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py

@@ -1,9 +1,7 @@
-from .canvas import MigrationCanvasCommand
 from .command import MigrationCommand
 from .prepare import MigrationPrepareCommand
 
 __all__ = [
-    "MigrationCanvasCommand",
     "MigrationCommand",
     "MigrationPrepareCommand",
 ]

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py

@@ -1,5 +1,6 @@
 from abc import ABC, abstractmethod
-from collections.abc import Sequence
+from collections import defaultdict
+from collections.abc import Callable, Sequence
 from typing import Generic, cast
 from uuid import uuid4
 
@@ -11,8 +12,15 @@ from cognite.client.data_classes.data_modeling import (
     View,
     ViewId,
 )
+from cognite.client.exceptions import CogniteException
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.client.data_classes.canvas import (
+    ContainerReferenceApply,
+    FdmInstanceContainerReferenceApply,
+    IndustrialCanvas,
+    IndustrialCanvasApply,
+)
 from cognite_toolkit._cdf_tk.client.data_classes.charts import Chart, ChartWrite
 from cognite_toolkit._cdf_tk.client.data_classes.charts_data import (
     ChartCoreTimeseries,
@@ -23,13 +31,18 @@ from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMa
 from cognite_toolkit._cdf_tk.commands._migrate.conversion import DirectRelationCache, asset_centric_to_dm
 from cognite_toolkit._cdf_tk.commands._migrate.data_classes import AssetCentricMapping
 from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_default_mappings
-from cognite_toolkit._cdf_tk.commands._migrate.issues import ChartMigrationIssue, ConversionIssue, MigrationIssue
+from cognite_toolkit._cdf_tk.commands._migrate.issues import (
+    CanvasMigrationIssue,
+    ChartMigrationIssue,
+    ConversionIssue,
+    MigrationIssue,
+)
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import AssetCentricMigrationSelector
 from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
-from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
+from cognite_toolkit._cdf_tk.exceptions import ToolkitMigrationError, ToolkitValueError
 from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.storageio._base import T_Selector
-from cognite_toolkit._cdf_tk.storageio.selectors import ChartSelector
+from cognite_toolkit._cdf_tk.storageio.selectors import CanvasSelector, ChartSelector
 from cognite_toolkit._cdf_tk.utils import humanize_collection
 from cognite_toolkit._cdf_tk.utils.useful_types import (
     T_AssetCentricResourceExtended,
@@ -245,3 +258,130 @@ class ChartMapper(DataMapper[ChartSelector, Chart, ChartWrite]):
             new_source_item = ChartSource(id=cast(str, core_ts_item.id), type=cast(str, core_ts_item.type))
             updated_source_collection.append(new_source_item)
         return updated_source_collection
+
+
+class CanvasMapper(DataMapper[CanvasSelector, IndustrialCanvas, IndustrialCanvasApply]):
+    # Note sequences are not supported in Canvas, so we do not include them here.
+    asset_centric_resource_types = tuple(("asset", "event", "timeseries", "file"))
+    DEFAULT_ASSET_VIEW = ViewId("cdf_cdm", "CogniteAsset", "v1")
+    DEFAULT_EVENT_VIEW = ViewId("cdf_cdm", "CogniteActivity", "v1")
+    DEFAULT_FILE_VIEW = ViewId("cdf_cdm", "CogniteFile", "v1")
+    DEFAULT_TIMESERIES_VIEW = ViewId("cdf_cdm", "CogniteTimeSeries", "v1")
+
+    def __init__(self, client: ToolkitClient, dry_run: bool, skip_on_missing_ref: bool = False) -> None:
+        self.client = client
+        self.dry_run = dry_run
+        self.skip_on_missing_ref = skip_on_missing_ref
+
+    def map(self, source: Sequence[IndustrialCanvas]) -> Sequence[tuple[IndustrialCanvasApply | None, MigrationIssue]]:
+        self._populate_cache(source)
+        output: list[tuple[IndustrialCanvasApply | None, MigrationIssue]] = []
+        for item in source:
+            mapped_item, issue = self._map_single_item(item)
+            output.append((mapped_item, issue))
+        return output
+
+    @property
+    def lookup_methods(self) -> dict[str, Callable]:
+        return {
+            "asset": self.client.migration.lookup.assets,
+            "event": self.client.migration.lookup.events,
+            "timeseries": self.client.migration.lookup.time_series,
+            "file": self.client.migration.lookup.files,
+        }
+
+    def _populate_cache(self, source: Sequence[IndustrialCanvas]) -> None:
+        """Populate the internal cache with references from the source canvases.
+
+        Note that the consumption views are also cached as part of the timeseries lookup.
+        """
+        ids_by_type: dict[str, set[int]] = defaultdict(set)
+        for canvas in source:
+            for ref in canvas.container_references or []:
+                if ref.container_reference_type in self.asset_centric_resource_types:
+                    ids_by_type[ref.container_reference_type].add(ref.resource_id)
+
+        for resource_type, lookup_method in self.lookup_methods.items():
+            ids = ids_by_type.get(resource_type)
+            if ids:
+                lookup_method(list(ids))
+
+    def _get_node_id(self, resource_id: int, resource_type: str) -> NodeId | None:
+        """Look up the node ID for a given resource ID and type."""
+        try:
+            return self.lookup_methods[resource_type](resource_id)
+        except KeyError:
+            raise ToolkitValueError(f"Unsupported resource type '{resource_type}' for container reference migration.")
+
+    def _get_consumer_view_id(self, resource_id: int, resource_type: str) -> ViewId:
+        """Look up the consumer view ID for a given resource ID and type."""
+        lookup_map = {
+            "asset": (self.client.migration.lookup.assets.consumer_view, self.DEFAULT_ASSET_VIEW),
+            "event": (self.client.migration.lookup.events.consumer_view, self.DEFAULT_EVENT_VIEW),
+            "timeseries": (self.client.migration.lookup.time_series.consumer_view, self.DEFAULT_TIMESERIES_VIEW),
+            "file": (self.client.migration.lookup.files.consumer_view, self.DEFAULT_FILE_VIEW),
+        }
+        if lookup_tuple := lookup_map.get(resource_type):
+            method, fallback = lookup_tuple
+            return method(resource_id) or fallback
+
+        raise ToolkitValueError(f"Unsupported resource type '{resource_type}' for container reference migration.")
+
+    def _map_single_item(self, canvas: IndustrialCanvas) -> tuple[IndustrialCanvasApply | None, CanvasMigrationIssue]:
+        update = canvas.as_write()
+        issue = CanvasMigrationIssue(canvas_external_id=canvas.canvas.external_id, canvas_name=canvas.canvas.name)
+
+        remaining_container_references: list[ContainerReferenceApply] = []
+        new_fdm_references: list[FdmInstanceContainerReferenceApply] = []
+        for ref in update.container_references or []:
+            if ref.container_reference_type not in self.asset_centric_resource_types:
+                remaining_container_references.append(ref)
+                continue
+            node_id = self._get_node_id(ref.resource_id, ref.container_reference_type)
+            if node_id is None:
+                issue.missing_reference_ids.append(ref.as_asset_centric_id())
+            else:
+                consumer_view = self._get_consumer_view_id(ref.resource_id, ref.container_reference_type)
+                fdm_ref = self.migrate_container_reference(ref, canvas.canvas.external_id, node_id, consumer_view)
+                new_fdm_references.append(fdm_ref)
+        if issue.missing_reference_ids and self.skip_on_missing_ref:
+            return None, issue
+
+        update.container_references = remaining_container_references
+        update.fdm_instance_container_references.extend(new_fdm_references)
+        if not self.dry_run:
+            backup = canvas.as_write().create_backup()
+            try:
+                self.client.canvas.industrial.create(backup)
+            except CogniteException as e:
+                raise ToolkitMigrationError(
+                    f"Failed to create backup for canvas '{canvas.canvas.name}': {e!s}. "
+                ) from e
+
+        return update, issue
+
+    @classmethod
+    def migrate_container_reference(
+        cls, reference: ContainerReferenceApply, canvas_external_id: str, instance_id: NodeId, consumer_view: ViewId
+    ) -> FdmInstanceContainerReferenceApply:
+        """Migrate a single container reference by replacing the asset-centric ID with the data model instance ID."""
+        new_id = str(uuid4())
+        new_external_id = f"{canvas_external_id}_{new_id}"
+        return FdmInstanceContainerReferenceApply(
+            external_id=new_external_id,
+            id_=new_id,
+            container_reference_type="fdmInstance",
+            instance_space=instance_id.space,
+            instance_external_id=instance_id.external_id,
+            view_space=consumer_view.space,
+            view_external_id=consumer_view.external_id,
+            view_version=consumer_view.version,
+            label=reference.label,
+            properties_=reference.properties_,
+            x=reference.x,
+            y=reference.y,
+            width=reference.width,
+            height=reference.height,
+            max_width=reference.max_width,
+            max_height=reference.max_height,
+        )

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/cognite_toolkit/_cdf_tk/commands/_migrate/issues.py

@@ -51,6 +51,18 @@ class ChartMigrationIssue(MigrationIssue):
         )
 
 
+class CanvasMigrationIssue(MigrationIssue):
+    type: ClassVar[str] = "canvasMigration"
+    canvas_external_id: str
+    canvas_name: str
+    missing_reference_ids: list[AssetCentricId] = Field(default_factory=list)
+
+    @property
+    def has_issues(self) -> bool:
+        """Check if there are any issues recorded in this CanvasMigrationIssue."""
+        return bool(self.missing_reference_ids)
+
+
 class ReadIssue(MigrationIssue):
     """Represents a read issue encountered during migration."""
 

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/cognite_toolkit/_cdf_tk/cruds/_base_cruds.py

@@ -1,6 +1,6 @@
 import sys
 from abc import ABC, abstractmethod
-from collections.abc import Hashable, Iterable, Sequence, Set, Sized
+from collections.abc import Hashable, Iterable, Sequence, Sized
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generic, TypeVar
 
@@ -9,7 +9,7 @@ from cognite.client.utils.useful_types import SequenceNotStr
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING, EXCL_FILES
+from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING, YAML_SUFFIX
 from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.resource_classes import ToolkitResource
 from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning
@@ -39,11 +39,9 @@ class Loader(ABC):
         exclude_filetypes: A set of filetypes that should be excluded from the supported filetypes.
     """
 
-    filetypes: frozenset[str]
     folder_name: str
     kind: str
     dependencies: "frozenset[type[ResourceCRUD]]" = frozenset()
-    exclude_filetypes: frozenset[str] = frozenset()
     _doc_base_url: str = "https://api-docs.cognite.com/20230101/tag/"
     _doc_url: str = ""
     sub_folder_name: str | None = None
@@ -74,13 +72,12 @@ class Loader(ABC):
     def doc_url(cls) -> str:
         return cls._doc_base_url + cls._doc_url
 
-    def find_files(self, dir_or_file: Path | None = None, include_formats: Set[str] | None = None) -> list[Path]:
+    def find_files(self, dir_or_file: Path | None = None) -> list[Path]:
         """Find all files that are supported by this loader in the given directory or file.
 
         Args:
             dir_or_file (Path): The directory or file to search in. If no path is given,
                 the build directory is used.
-            include_formats (set[str]): A set of file formats to include. If not set, all formats are included.
 
         Returns:
             list[Path]: A sorted list of all files that are supported by this loader.
@@ -94,12 +91,7 @@ class Loader(ABC):
                 raise ValueError("Invalid file type")
             return [dir_or_file]
         elif dir_or_file.is_dir():
-            file_paths = [
-                file
-                for file in dir_or_file.glob("**/*")
-                if self.is_supported_file(file) and (include_formats is None or file.suffix in include_formats)
-            ]
-            return sorted(file_paths)
+            return sorted([file for file in dir_or_file.rglob("*") if self.is_supported_file(file)])
         else:
             return []
 
@@ -118,11 +110,7 @@ class Loader(ABC):
             bool: True if the file is supported, False otherwise.
 
         """
-        if cls.filetypes and file.suffix[1:] not in cls.filetypes:
-            return False
-        if cls.exclude_filetypes and file.suffix[1:] in cls.exclude_filetypes:
-            return False
-        return file.stem.casefold().endswith(cls.kind.casefold())
+        return file.suffix in YAML_SUFFIX and file.stem.casefold().endswith(cls.kind.casefold())
 
 
 T_Loader = TypeVar("T_Loader", bound=Loader)
@@ -144,8 +132,6 @@ class ResourceCRUD(Loader, ABC, Generic[T_ID, T_ResourceRequest, T_ResourceRespo
         yaml_cls: The File format for this resource. This is used to validate the user input.
         support_drop: Whether the resource supports the drop flag.
         support_update: Whether the resource supports the update operation.
-        filetypes: The filetypes that are supported by this crud. This should not be set in the subclass, it
-            should always be yaml and yml.
         dependencies: A set of other resource cruds that must be loaded before this crud.
         parent_resource: A set of other resource cruds that are parent resources to this resource. This is used
             to determine if the iterate method should return any resources when filtering by parent ids.
@@ -158,7 +144,6 @@ class ResourceCRUD(Loader, ABC, Generic[T_ID, T_ResourceRequest, T_ResourceRespo
     # Optional to set in the subclass
     support_drop = True
     support_update = True
-    filetypes = frozenset({"yaml", "yml"})
     dependencies: "frozenset[type[ResourceCRUD]]" = frozenset()
     # For example, TransformationNotification and Schedule has Transformation as the parent resource
     # This is used in the iterate method to ensure that nothing is returned if
@@ -437,10 +422,3 @@ class DataCRUD(Loader, ABC):
     @abstractmethod
     def upload(self, state: "BuildEnvironment", dry_run: bool) -> Iterable[tuple[str, int]]:
         raise NotImplementedError
-
-    def _find_data_files(self, directory: Path) -> list[Path]:
-        return [
-            path
-            for path in directory.rglob("*")
-            if path.is_file() and path.name not in EXCL_FILES and self.is_supported_file(path)
-        ]

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/cognite_toolkit/_cdf_tk/cruds/_data_cruds.py

@@ -28,7 +28,6 @@ class DatapointsCRUD(DataCRUD):
     item_name = "datapoints"
     folder_name = "timeseries"
     kind = "Datapoints"
-    filetypes = frozenset({"csv", "parquet"})
     dependencies = frozenset({TimeSeriesCRUD})
     _doc_url = "Time-series/operation/postMultiTimeSeriesDatapoints"
 
@@ -43,7 +42,9 @@ class DatapointsCRUD(DataCRUD):
         resource_directories = state.built_resources[self.folder_name].get_resource_directories(self.folder_name)
 
         for resource_dir in resource_directories:
-            for datafile in self._find_data_files(resource_dir):
+            for datafile in resource_dir.rglob("*"):
+                if not datafile.stem.casefold().endswith(self.kind.casefold()):
+                    continue
                 if datafile.suffix == ".csv":
                     # The replacement is used to ensure that we read exactly the same file on Windows and Linux
                     file_content = datafile.read_bytes().replace(b"\r\n", b"\n").decode("utf-8")
@@ -52,7 +53,7 @@ class DatapointsCRUD(DataCRUD):
                 elif datafile.suffix == ".parquet":
                     data = pd.read_parquet(datafile, engine="pyarrow")
                 else:
-                    raise ValueError(f"Unsupported file type {datafile.suffix} for {datafile.name}")
+                    continue
                 timeseries_ids = list(data.columns)
                 if len(timeseries_ids) == 1:
                     ts_str = timeseries_ids[0]
@@ -87,8 +88,6 @@ class FileCRUD(DataCRUD):
     item_name = "file contents"
     folder_name = "files"
     kind = "File"
-    filetypes = frozenset()
-    exclude_filetype: frozenset[str] = frozenset({})
     dependencies = frozenset({FileMetadataCRUD, CogniteFileCRUD})
     _doc_url = "Files/operation/initFileUpload"
 
@@ -149,7 +148,6 @@ class FileCRUD(DataCRUD):
 class RawFileCRUD(DataCRUD):
     item_name = "rows"
     folder_name = "raw"
-    filetypes = frozenset({"csv", "parquet"})
     kind = "Raw"
     dependencies = frozenset({RawTableCRUD})
     _doc_url = "Raw/operation/postRows"
@@ -169,7 +167,7 @@ class RawFileCRUD(DataCRUD):
             datafile = next(
                 (
                     resource.source.path.with_suffix(f".{file_type}")
-                    for file_type in self.filetypes
+                    for file_type in ["csv", "parquet"]
                     if (resource.source.path.with_suffix(f".{file_type}").exists())
                 ),
                 None,

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py

@@ -29,20 +29,23 @@ from cognite_toolkit._cdf_tk.client.data_classes.sequences import (
     ToolkitSequenceRowsList,
     ToolkitSequenceRowsWrite,
 )
+from cognite_toolkit._cdf_tk.constants import TABLE_FORMATS, YAML_SUFFIX
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.resource_classes import AssetYAML, EventYAML, SequenceRowYAML, SequenceYAML
-from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning
+from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning, ToolkitDeprecationWarning
 from cognite_toolkit._cdf_tk.utils import load_yaml_inject_variables
 from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_hashable, diff_list_identifiable
 from cognite_toolkit._cdf_tk.utils.file import read_csv
 
 from .data_organization import DataSetsCRUD, LabelCRUD
 
+_DEPRECATION_WARNING_ISSUED = False
+
 
 @final
 class AssetCRUD(ResourceCRUD[str, AssetWrite, Asset]):
     folder_name = "classic"
-    filetypes = frozenset({"yaml", "yml", "csv", "parquet"})
     resource_cls = Asset
     resource_write_cls = AssetWrite
     yaml_cls = AssetYAML
@@ -50,6 +53,26 @@ class AssetCRUD(ResourceCRUD[str, AssetWrite, Asset]):
     dependencies = frozenset({DataSetsCRUD, LabelCRUD})
     _doc_url = "Assets/operation/createAssets"
 
+    @classmethod
+    def is_supported_file(cls, file: Path) -> bool:
+        if Flags.v08.is_enabled():
+            return super().is_supported_file(file)
+        global _DEPRECATION_WARNING_ISSUED
+        if not file.stem.casefold().endswith(cls.kind.casefold()):
+            return False
+        if file.suffix in YAML_SUFFIX:
+            return True
+        if file.suffix in TABLE_FORMATS:
+            if not _DEPRECATION_WARNING_ISSUED:
+                ToolkitDeprecationWarning(
+                    feature="deployment of asset from CSV or Parquet files",
+                    alternative="data plugin and cdf data upload commands",
+                    removal_version="0.8",
+                ).print_warning()
+                _DEPRECATION_WARNING_ISSUED = True
+            return True
+        return False
+
     @property
     def display_name(self) -> str:
         return "assets"
@@ -479,7 +502,6 @@ class SequenceRowCRUD(ResourceCRUD[str, ToolkitSequenceRowsWrite, ToolkitSequenc
 @final
 class EventCRUD(ResourceCRUD[str, EventWrite, Event]):
     folder_name = "classic"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = Event
     resource_write_cls = EventWrite
     yaml_cls = EventYAML

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py

@@ -40,7 +40,6 @@ from .group_scoped import GroupResourceScopedCRUD
 @final
 class InfieldV1CRUD(ResourceCRUD[str, APMConfigWrite, APMConfig]):
     folder_name = "cdf_applications"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = APMConfig
     resource_write_cls = APMConfigWrite
     kind = "InfieldV1"
@@ -249,7 +248,6 @@ class InfieldV1CRUD(ResourceCRUD[str, APMConfigWrite, APMConfig]):
 @final
 class InFieldLocationConfigCRUD(ResourceCRUD[NodeIdentifier, InfieldLocationConfig, InfieldLocationConfig]):
     folder_name = "cdf_applications"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = InfieldLocationConfig
     resource_write_cls = InfieldLocationConfig
     kind = "InFieldLocationConfig"
@@ -349,7 +347,6 @@ class InFieldLocationConfigCRUD(ResourceCRUD[NodeIdentifier, InfieldLocationConf
 @final
 class InFieldCDMLocationConfigCRUD(ResourceCRUD[NodeIdentifier, InFieldCDMLocationConfig, InFieldCDMLocationConfig]):
     folder_name = "cdf_applications"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = InFieldCDMLocationConfig
     resource_write_cls = InFieldCDMLocationConfig
     kind = "InFieldCDMLocationConfig"

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/migration.py

@@ -21,7 +21,6 @@ from .datamodel import SpaceCRUD, ViewCRUD
 @final
 class ResourceViewMappingCRUD(ResourceCRUD[str, ResourceViewMappingApply, ResourceViewMapping]):
     folder_name = "migration"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = ResourceViewMapping
     resource_write_cls = ResourceViewMappingApply
     kind = "ResourceViewMapping"

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/relationship.py

@@ -23,7 +23,6 @@ from .timeseries import TimeSeriesCRUD
 @final
 class RelationshipCRUD(ResourceCRUD[str, RelationshipWrite, Relationship]):
     folder_name = "classic"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = Relationship
     resource_write_cls = RelationshipWrite
     kind = "Relationship"

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py

@@ -19,7 +19,6 @@ from .datamodel import ContainerCRUD
 @final
 class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse]):
     folder_name = "streams"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = StreamResponse
     resource_write_cls = StreamRequest
     kind = "Streams"

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.27}/cognite_toolkit/_cdf_tk/cruds/_worker.py

@@ -14,6 +14,7 @@ from yaml import YAMLError
 
 from cognite_toolkit._cdf_tk.constants import TABLE_FORMATS
 from cognite_toolkit._cdf_tk.exceptions import ToolkitWrongResourceError, ToolkitYAMLFormatError
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarning, catch_warnings
 from cognite_toolkit._cdf_tk.utils import to_diff
 
@@ -47,12 +48,16 @@ class ResourceWorker(Generic[T_ID, T_ResourceRequest, T_ResourceResponse]):
     ) -> list[Path]:
         filepaths = self.loader.find_files(directory)
 
-        for read_module in read_modules or []:
-            if resource_dir := read_module.resource_dir_path(self.loader.folder_name):
-                # As of 05/11/24, Asset support csv and parquet files in addition to YAML.
-                # These table formats are not built, i.e., no variable replacement is done,
-                # so we load them directly from the source module.
-                filepaths.extend(self.loader.find_files(resource_dir, include_formats=TABLE_FORMATS))
+        if not Flags.v08.is_enabled():
+            for read_module in read_modules or []:
+                if resource_dir := read_module.resource_dir_path(self.loader.folder_name):
+                    # As of 05/11/24, Asset support csv and parquet files in addition to YAML.
+                    # These table formats are not built, i.e., no variable replacement is done,
+                    # so we load them directly from the source module.
+                    table_files = [
+                        file for file in self.loader.find_files(resource_dir) if file.suffix in TABLE_FORMATS
+                    ]
+                    filepaths.extend(table_files)
 
         if not sort:
             return filepaths