cognite-toolkit 0.5.63__py3-none-any.whl → 0.5.65__py3-none-any.whl

cognite_toolkit/_builtin_modules/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.5.63"
+version = "0.5.65"
 
 
 [plugins]

cognite_toolkit/_cdf_tk/apps/_migrate_app.py

@@ -3,7 +3,7 @@ from typing import Annotated, Any
 
 import typer
 
-from cognite_toolkit._cdf_tk.commands import MigrateTimeseriesCommand, MigrationPrepareCommand
+from cognite_toolkit._cdf_tk.commands import MigrateAssetsCommand, MigrateTimeseriesCommand, MigrationPrepareCommand
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 
 
@@ -12,6 +12,8 @@ class MigrateApp(typer.Typer):
         super().__init__(*args, **kwargs)
         self.callback(invoke_without_command=True)(self.main)
         self.command("prepare")(self.prepare)
+        # Uncomment when command is ready.
+        # self.command("assets")(self.assets)
         self.command("timeseries")(self.timeseries)
 
     def main(self, ctx: typer.Context) -> None:
@@ -57,6 +59,48 @@ class MigrateApp(typer.Typer):
             )
         )
 
+    @staticmethod
+    def assets(
+        ctx: typer.Context,
+        mapping_file: Annotated[
+            Path,
+            typer.Option(
+                "--mapping-file",
+                "-m",
+                help="Path to the mapping file that contains the mapping from Assets to CogniteAssets. "
+                "This file is expected to have the following columns: [id/externalId, dataSetId, space, externalId]."
+                "The dataSetId is optional, and can be skipped. If it is set, it is used to check the access to the dataset.",
+            ),
+        ],
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the migration will not be executed, but only a report of what would be done is printed.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Migrate Assets to CogniteAssets."""
+        client = EnvironmentVariables.create_from_environment().get_client()
+        cmd = MigrateAssetsCommand()
+        cmd.run(
+            lambda: cmd.migrate_assets(
+                client,
+                mapping_file=mapping_file,
+                dry_run=dry_run,
+                verbose=verbose,
+            )
+        )
+
     @staticmethod
     def timeseries(
         ctx: typer.Context,

cognite_toolkit/_cdf_tk/client/api/extended_timeseries.py

@@ -1,6 +1,7 @@
 from collections.abc import Sequence
-from typing import Any, overload
+from typing import Any, cast, overload
 
+from cognite.client import ClientConfig, CogniteClient
 from cognite.client._api.time_series import SortSpec, TimeSeriesAPI
 from cognite.client._constants import DEFAULT_LIMIT_READ
 from cognite.client.data_classes.data_modeling import NodeId
@@ -19,6 +20,11 @@ from cognite_toolkit._cdf_tk.client.data_classes.pending_instances_ids import Pe
 class ExtendedTimeSeriesAPI(TimeSeriesAPI):
     """Extended TimeSeriesAPI to include pending ID methods."""
 
+    def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: CogniteClient) -> None:
+        super().__init__(config, api_version, cognite_client)
+        self._PENDING_IDS_LIMIT = 1000
+        self._UNLINK_LIMIT = 1000
+
     @overload
     def set_pending_ids(
         self, instance_id: NodeId | tuple[str, str], id: int | None = None, external_id: str | None = None
@@ -74,7 +80,7 @@ class ExtendedTimeSeriesAPI(TimeSeriesAPI):
                 },
                 "api_subversion": "alpha",
             }
-            for id_chunk in split_into_chunks(list(identifiers), 1000)
+            for id_chunk in split_into_chunks(list(identifiers), self._PENDING_IDS_LIMIT)
         ]
         tasks_summary = execute_tasks(
             self._post,
@@ -90,6 +96,68 @@ class ExtendedTimeSeriesAPI(TimeSeriesAPI):
 
         return ExtendedTimeSeriesList._load(retrieved_items, cognite_client=self._cognite_client)
 
+    @overload
+    def unlink_instance_ids(
+        self,
+        id: int | None = None,
+        external_id: str | None = None,
+    ) -> ExtendedTimeSeries | None: ...
+
+    @overload
+    def unlink_instance_ids(
+        self,
+        id: Sequence[int] | None = None,
+        external_id: SequenceNotStr[str] | None = None,
+    ) -> ExtendedTimeSeriesList: ...
+
+    def unlink_instance_ids(
+        self,
+        id: int | Sequence[int] | None = None,
+        external_id: str | SequenceNotStr[str] | None = None,
+    ) -> ExtendedTimeSeries | ExtendedTimeSeriesList | None:
+        """Unlink pending instance IDs from time series.
+
+        Args:
+            id (int | Sequence[int] | None): The ID(s) of the time series.
+            external_id (str | SequenceNotStr[str] | None): The external ID(s) of the time series.
+
+        """
+        if id is None and external_id is None:
+            return None
+        if isinstance(id, int) and isinstance(external_id, str):
+            raise ValueError("Cannot specify both id and external_id as single values. Use one or the other.")
+        is_single = isinstance(id, int) or isinstance(external_id, str)
+        identifiers = IdentifierSequence.load(id, external_id)
+
+        tasks = [
+            {
+                "url_path": f"{self._RESOURCE_PATH}/unlink-instance-ids",
+                "json": {"items": id_chunk},
+                "api_subversion": "alpha",
+            }
+            for id_chunk in split_into_chunks(identifiers.as_dicts(), self._UNLINK_LIMIT)
+        ]
+        tasks_summary = execute_tasks(
+            self._post,
+            tasks,
+            max_workers=self._config.max_workers,
+            fail_fast=True,
+        )
+        tasks_summary.raise_compound_exception_if_failed_tasks(
+            task_unwrap_fn=unpack_items_in_payload,
+        )
+
+        retrieved_items = tasks_summary.joined_results(lambda res: res.json()["items"])
+
+        result = ExtendedTimeSeriesList._load(retrieved_items, cognite_client=self._cognite_client)
+        if is_single:
+            if len(result) == 0:
+                return None
+            if len(result) > 1:
+                raise ValueError("Expected a single time series, but multiple were returned.")
+            return cast(ExtendedTimeSeries, result[0])
+        return cast(ExtendedTimeSeriesList, result)
+
     def retrieve(
         self, id: int | None = None, external_id: str | None = None, instance_id: NodeId | None = None
     ) -> ExtendedTimeSeries | None:

cognite_toolkit/_cdf_tk/commands/__init__.py

@@ -1,4 +1,4 @@
-from ._migrate import MigrateTimeseriesCommand, MigrationPrepareCommand
+from ._migrate import MigrateAssetsCommand, MigrateTimeseriesCommand, MigrationPrepareCommand
 from ._populate import PopulateCommand
 from ._profile import ProfileAssetCentricCommand, ProfileTransformationCommand
 from ._purge import PurgeCommand
@@ -26,6 +26,7 @@ __all__ = [
     "DumpResourceCommand",
     "FeatureFlagCommand",
     "InitCommand",
+    "MigrateAssetsCommand",
     "MigrateTimeseriesCommand",
     "MigrationPrepareCommand",
     "ModulesCommand",

cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py

@@ -1,4 +1,5 @@
+from .assets import MigrateAssetsCommand
 from .prepare import MigrationPrepareCommand
 from .timeseries import MigrateTimeseriesCommand
 
-__all__ = ["MigrateTimeseriesCommand", "MigrationPrepareCommand"]
+__all__ = ["MigrateAssetsCommand", "MigrateTimeseriesCommand", "MigrationPrepareCommand"]

cognite_toolkit/_cdf_tk/commands/_migrate/assets.py

@@ -0,0 +1,201 @@
+from collections.abc import Callable, Iterable
+from pathlib import Path
+
+from cognite.client.data_classes import Asset, Label, LabelDefinition
+from cognite.client.data_classes.capabilities import (
+    Capability,
+    DataModelInstancesAcl,
+    DataModelsAcl,
+    DataSetScope,
+    SpaceIDScope,
+    TimeSeriesAcl,
+)
+from cognite.client.data_classes.data_modeling import NodeApply, NodeOrEdgeData, ViewId
+from cognite.client.exceptions import CogniteAPIError, CogniteException
+from rich import print
+
+from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.client._constants import DATA_MODELING_MAX_WRITE_WORKERS
+from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
+from cognite_toolkit._cdf_tk.constants import DMS_INSTANCE_LIMIT_MARGIN
+from cognite_toolkit._cdf_tk.exceptions import (
+    AuthenticationError,
+    ResourceCreationError,
+    ResourceRetrievalError,
+    ToolkitMigrationError,
+    ToolkitValueError,
+)
+from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning
+from cognite_toolkit._cdf_tk.utils import humanize_collection
+from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
+from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
+
+from .data_classes import MigrationMapping, MigrationMappingList
+from .data_model import MAPPING_VIEW_ID
+
+
+class MigrateAssetsCommand(ToolkitCommand):
+    cdf_cdm = "cdf_cdm"
+    asset_id = ViewId(cdf_cdm, "CogniteAsset", "v1")
+
+    # This is the number of timeseries that can be written in parallel.
+    chunk_size = 1000 * DATA_MODELING_MAX_WRITE_WORKERS
+
+    def migrate_assets(
+        self,
+        client: ToolkitClient,
+        mapping_file: Path,
+        dry_run: bool = False,
+        verbose: bool = False,
+    ) -> None:
+        """Migrate resources from Asset-Centric to data modeling in CDF."""
+        mappings = MigrationMappingList.read_mapping_file(mapping_file)
+        self._validate_access(client, mappings)
+        self._validate_migration_mappings_exists(client)
+        self._validate_available_capacity(client, mappings)
+        iteration_count = len(mappings) // self.chunk_size + 1
+        executor = ProducerWorkerExecutor[list[tuple[Asset, MigrationMapping]], list[NodeApply]](
+            download_iterable=self._download_assets(client, mappings),
+            process=self._as_cognite_assets,
+            write=self._upload_assets(client, dry_run=dry_run, verbose=verbose),
+            iteration_count=iteration_count,
+            max_queue_size=10,
+            download_description="Downloading assets",
+            process_description="Converting assets to CogniteAssets",
+            write_description="Uploading CogniteAssets",
+        )
+        executor.run()
+        if executor.error_occurred:
+            raise ResourceCreationError(executor.error_message)
+
+        prefix = "Would have" if dry_run else "Successfully"
+        self.console(f"{prefix} migrated {executor.total_items:,} assets to CogniteAssets.")
+
+    def _validate_access(self, client: ToolkitClient, mappings: MigrationMappingList) -> None:
+        required_capabilities: list[Capability] = [
+            DataModelsAcl(
+                actions=[DataModelsAcl.Action.Read], scope=SpaceIDScope([self.cdf_cdm, MAPPING_VIEW_ID.space])
+            ),
+            DataModelInstancesAcl(
+                actions=[
+                    DataModelInstancesAcl.Action.Read,
+                    DataModelInstancesAcl.Action.Write,
+                    DataModelInstancesAcl.Action.Write_Properties,
+                ],
+                scope=SpaceIDScope(list(mappings.spaces())),
+            ),
+        ]
+        if data_set_ids := mappings.get_data_set_ids():
+            required_capabilities.append(
+                TimeSeriesAcl(
+                    actions=[TimeSeriesAcl.Action.Read, TimeSeriesAcl.Action.Write],
+                    scope=DataSetScope(list(data_set_ids)),
+                )
+            )
+        if missing := client.iam.verify_capabilities(required_capabilities):
+            raise AuthenticationError(f"Missing required capabilities: {humanize_collection(missing)}.")
+
+    def _validate_migration_mappings_exists(self, client: ToolkitClient) -> None:
+        view = client.data_modeling.views.retrieve(MAPPING_VIEW_ID)
+        if not view:
+            raise ToolkitMigrationError(
+                f"The migration mapping view {MAPPING_VIEW_ID} does not exist. "
+                f"Please run the `cdf migrate prepare` command to deploy the migration data model."
+            )
+
+    def _validate_available_capacity(self, client: ToolkitClient, mappings: MigrationMappingList) -> None:
+        """Validate that the project has enough capacity to accommodate the migration."""
+        try:
+            stats = client.data_modeling.statistics.project()
+        except CogniteAPIError:
+            # This endpoint is not yet in alpha, it may change or not be available.
+            self.warn(HighSeverityWarning("Cannot check the instances capacity proceeding with migration anyway."))
+            return
+        available_capacity = stats.instances.instances_limit - stats.instances.instances
+        available_capacity_after = available_capacity - len(mappings)
+
+        if available_capacity_after < DMS_INSTANCE_LIMIT_MARGIN:
+            raise ToolkitValueError(
+                "Cannot proceed with migration, not enough instance capacity available. Total capacity after migration"
+                f"would be {available_capacity_after:,} instances, which is less than the required margin of"
+                f"{DMS_INSTANCE_LIMIT_MARGIN:,} instances. Please increase the instance capacity in your CDF project"
+                f" or delete some existing instances before proceeding with the migration of {len(mappings):,} assets."
+            )
+        total_instances = stats.instances.instances + len(mappings)
+        self.console(
+            f"Project has enough capacity for migration. Total instances after migration: {total_instances:,}."
+        )
+
+    def _download_assets(
+        self, client: ToolkitClient, mappings: MigrationMappingList
+    ) -> Iterable[list[tuple[Asset, MigrationMapping]]]:
+        for chunk in chunker_sequence(mappings, self.chunk_size):
+            try:
+                asset_list = client.assets.retrieve_multiple(
+                    chunk.get_ids(), chunk.get_external_ids(), ignore_unknown_ids=True
+                )
+            except CogniteException as e:
+                raise ResourceRetrievalError(f"Failed to retrieve {len(chunk):,} assets: {e!s}") from e
+            mapping_by_id = chunk.as_mapping_by_id()
+            chunk_list: list[tuple[Asset, MigrationMapping]] = []
+            for asset in asset_list:
+                if asset.id in mapping_by_id:
+                    chunk_list.append((asset, mapping_by_id[asset.id]))
+                elif asset.external_id in mapping_by_id:
+                    chunk_list.append((asset, mapping_by_id[asset.external_id]))
+            yield chunk_list
+
+    def _as_cognite_assets(self, assets: list[tuple[Asset, MigrationMapping]]) -> list[NodeApply]:
+        """Convert Asset objects to CogniteAssetApply objects."""
+        return [self.as_cognite_asset(asset, mapping) for asset, mapping in assets]
+
+    @classmethod
+    def _upload_assets(cls, client: ToolkitClient, dry_run: bool, verbose: bool) -> Callable[[list[NodeApply]], None]:
+        def upload_assets(assets: list[NodeApply]) -> None:
+            if dry_run:
+                if verbose:
+                    print(f"Would have created {len(assets):,} CogniteAssets.")
+                return
+            try:
+                created = client.data_modeling.instances.apply_fast(assets)
+            except CogniteException as e:
+                raise ResourceCreationError(f"Failed to upsert CogniteAssets {len(assets):,}: {e!s}") from e
+            if verbose:
+                print(f"Created {len(created):,} CogniteAssets.")
+
+        return upload_assets
+
+    @classmethod
+    def as_cognite_asset(cls, asset: Asset, mapping: MigrationMapping) -> NodeApply:
+        tags: list[str] = []
+        for label in asset.labels or []:
+            if isinstance(label, str):
+                tags.append(label)
+            elif isinstance(label, dict) and "externalId" in label:
+                tags.append(label["externalId"])
+            elif isinstance(label, Label | LabelDefinition) and label.external_id:
+                tags.append(label.external_id)
+
+        return NodeApply(
+            space=mapping.instance_id.space,
+            external_id=mapping.instance_id.external_id,
+            sources=[
+                NodeOrEdgeData(
+                    source=cls.asset_id,
+                    properties={
+                        "name": asset.name,
+                        "description": asset.description,
+                        "tags": tags or None,
+                    },
+                ),
+                NodeOrEdgeData(
+                    source=MAPPING_VIEW_ID,
+                    properties={
+                        "resourceType": "asset",
+                        "id": asset.id,
+                        "dataSetId": asset.data_set_id,
+                        "classicExternalId": asset.external_id,
+                    },
+                ),
+            ],
+        )

cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py

@@ -1,5 +1,6 @@
 import csv
 import sys
+from abc import abstractmethod
 from collections.abc import Collection, Iterator, Sequence
 from dataclasses import dataclass
 from pathlib import Path
@@ -24,18 +25,28 @@ class MigrationMapping:
     resource_type: str
     instance_id: NodeId
 
+    @abstractmethod
+    def get_id(self) -> int | str:
+        raise NotImplementedError()
+
 
 @dataclass
 class IdMigrationMapping(MigrationMapping):
     id: int
     data_set_id: int | None = None
 
+    def get_id(self) -> int:
+        return self.id
+
 
 @dataclass
 class ExternalIdMigrationMapping(MigrationMapping):
     external_id: str
     data_set_id: int | None = None
 
+    def get_id(self) -> str:
+        return self.external_id
+
 
 class MigrationMappingList(list, Sequence[MigrationMapping]):
     # Implemented to get correct type hints
@@ -90,6 +101,10 @@ class MigrationMappingList(list, Sequence[MigrationMapping]):
             if isinstance(mapping, IdMigrationMapping | ExternalIdMigrationMapping) and mapping.data_set_id is not None
         }
 
+    def as_mapping_by_id(self) -> dict[int | str, MigrationMapping]:
+        """Return a mapping of IDs to MigrationMapping objects."""
+        return {mapping.get_id(): mapping for mapping in self}
+
     @classmethod
     def read_mapping_file(cls, mapping_file: Path) -> Self:
         if not mapping_file.exists():

cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py

@@ -66,6 +66,8 @@ MAPPING_VIEW = dm.ViewApply(
     },
 )
 
+MAPPING_VIEW_ID = MAPPING_VIEW.as_id()
+
 COGNITE_MIGRATION_MODEL = dm.DataModelApply(
     space=SPACE.space,
     external_id="CogniteMigration",

cognite_toolkit/_cdf_tk/commands/_migrate/prepare.py

@@ -25,7 +25,7 @@ class MigrationPrepareCommand(ToolkitCommand):
         verb = "Would deploy" if dry_run else "Deploying"
         print(f"{verb} {MODEL_ID!r}")
         results = DeployResults([], "deploy", dry_run=dry_run)
-        for loader_cls, resources in [
+        for loader_cls, resource_list in [
             (SpaceLoader, [SPACE]),
             (ContainerLoader, [MAPPING_CONTAINER]),
             (ViewLoader, [MAPPING_VIEW]),
@@ -35,23 +35,15 @@ class MigrationPrepareCommand(ToolkitCommand):
             loader = loader_cls.create_loader(client)  # type: ignore[attr-defined]
             worker = ResourceWorker(loader)
             # MyPy does not understand that `loader` has a `get_id` method.
-            local_by_id = {loader.get_id(item): (item.dump(), item) for item in resources}  # type: ignore[attr-defined]
+            local_by_id = {loader.get_id(item): (item.dump(), item) for item in resource_list}  # type: ignore[attr-defined]
             worker.validate_access(local_by_id, is_dry_run=dry_run)
             cdf_resources = loader.retrieve(list(local_by_id.keys()))
-            to_create, to_update, to_delete, unchanged = worker.categorize_resources(
-                local_by_id, cdf_resources, False, verbose
-            )
+            resources = worker.categorize_resources(local_by_id, cdf_resources, False, verbose)
 
             if dry_run:
-                result = deploy_cmd.dry_run_deploy(to_create, to_update, to_delete, unchanged, loader, False, False)
+                result = deploy_cmd.dry_run_deploy(resources, loader, False, False)
             else:
-                result = deploy_cmd.actual_deploy(
-                    to_create,
-                    to_update,
-                    to_delete,
-                    unchanged,
-                    loader,
-                )
+                result = deploy_cmd.actual_deploy(resources, loader)
             if result:
                 results[result.name] = result
         if results.has_counts:

cognite_toolkit/_cdf_tk/commands/deploy.py

@@ -45,6 +45,7 @@ from cognite_toolkit._cdf_tk.loaders import (
     ResourceWorker,
 )
 from cognite_toolkit._cdf_tk.loaders._base_loaders import T_WritableCogniteResourceList
+from cognite_toolkit._cdf_tk.loaders._worker import CategorizedResources
 from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarning
 from cognite_toolkit._cdf_tk.tk_warnings.base import WarningList, catch_warnings
 from cognite_toolkit._cdf_tk.tk_warnings.other import (
@@ -261,7 +262,7 @@ class DeployCommand(ToolkitCommand):
             return None
 
         with catch_warnings(EnvironmentVariableMissingWarning) as env_var_warnings:
-            to_create, to_update, to_delete, unchanged = worker.prepare_resources(
+            resources = worker.prepare_resources(
                 files,
                 environment_variables=env_vars.dump(include_os=True),
                 is_dry_run=dry_run,
@@ -274,7 +275,7 @@ class DeployCommand(ToolkitCommand):
 
         # We are not counting to_delete as these are captured by to_create.
         # (to_delete is used for resources that does not support update and instead needs to be deleted and recreated)
-        nr_of_items = len(to_create) + len(to_update) + len(unchanged)
+        nr_of_items = len(resources.to_create) + len(resources.to_update) + len(resources.unchanged)
         if nr_of_items == 0:
             return ResourceDeployResult(name=loader.display_name)
 
@@ -288,19 +289,16 @@ class DeployCommand(ToolkitCommand):
 
         if dry_run:
             result = self.dry_run_deploy(
-                to_create,
-                to_update,
-                to_delete,
-                unchanged,
+                resources,
                 loader,
                 has_done_drop,
                 has_dropped_data,
             )
         else:
-            result = self.actual_deploy(to_create, to_update, to_delete, unchanged, loader, env_var_warnings)
+            result = self.actual_deploy(resources, loader, env_var_warnings)
 
         if verbose:
-            self._verbose_print(to_create, to_update, unchanged, loader, dry_run)
+            self._verbose_print(resources, loader, dry_run)
 
         if isinstance(loader, ResourceContainerLoader):
             return ResourceContainerDeployResult.from_resource_deploy_result(
@@ -311,10 +309,7 @@ class DeployCommand(ToolkitCommand):
 
     def actual_deploy(
         self,
-        to_create: T_CogniteResourceList,
-        to_update: T_CogniteResourceList,
-        to_delete: list[T_ID],
-        unchanged: T_CogniteResourceList,
+        resources: CategorizedResources[T_ID, T_CogniteResourceList],
         loader: ResourceLoader[
             T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
         ],
@@ -326,16 +321,16 @@ class DeployCommand(ToolkitCommand):
             if isinstance(warning, EnvironmentVariableMissingWarning)
             for identifier in warning.identifiers or []
         }
-        nr_of_unchanged = len(unchanged)
+        nr_of_unchanged = len(resources.unchanged)
         nr_of_deleted, nr_of_created, nr_of_changed = 0, 0, 0
-        if to_delete:
-            deleted = loader.delete(to_delete)
+        if resources.to_delete:
+            deleted = loader.delete(resources.to_delete)
             nr_of_deleted += deleted
-        if to_create:
-            created = self._create_resources(to_create, loader, environment_variable_warning_by_id)
+        if resources.to_create:
+            created = self._create_resources(resources.to_create, loader, environment_variable_warning_by_id)
             nr_of_created += created
-        if to_update:
-            updated = self._update_resources(to_update, loader, environment_variable_warning_by_id)
+        if resources.to_update:
+            updated = self._update_resources(resources.to_update, loader, environment_variable_warning_by_id)
             nr_of_changed += updated
         return ResourceDeployResult(
             name=loader.display_name,
@@ -348,10 +343,7 @@ class DeployCommand(ToolkitCommand):
 
     @staticmethod
     def dry_run_deploy(
-        to_create: T_CogniteResourceList,
-        to_update: T_CogniteResourceList,
-        to_delete: list[T_ID],
-        unchanged: T_CogniteResourceList,
+        resources: CategorizedResources[T_ID, T_CogniteResourceList],
         loader: ResourceLoader[
             T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
         ],
@@ -364,39 +356,40 @@ class DeployCommand(ToolkitCommand):
             and (not isinstance(loader, ResourceContainerLoader) or has_dropped_data)
         ):
             # Means the resources will be deleted and not left unchanged or changed
-            for item in unchanged:
+            for item in resources.unchanged:
                 # We cannot use extents as LoadableNodes cannot be extended.
-                to_create.append(item)
-            for item in to_update:
-                to_create.append(item)
-            unchanged.clear()
-            to_update.clear()
+                resources.to_create.append(item)
+            for item in resources.to_update:
+                resources.to_create.append(item)
+            resources.unchanged.clear()
+            resources.to_update.clear()
         return ResourceDeployResult(
             name=loader.display_name,
-            created=len(to_create),
-            deleted=len(to_delete),
-            changed=len(to_update),
-            unchanged=len(unchanged),
-            total=len(to_create) + len(to_delete) + len(to_update) + len(unchanged),
+            created=len(resources.to_create),
+            deleted=len(resources.to_delete),
+            changed=len(resources.to_update),
+            unchanged=len(resources.unchanged),
+            total=len(resources.to_create)
+            + len(resources.to_delete)
+            + len(resources.to_update)
+            + len(resources.unchanged),
         )
 
     @staticmethod
     def _verbose_print(
-        to_create: T_CogniteResourceList,
-        to_update: T_CogniteResourceList,
-        unchanged: T_CogniteResourceList,
+        resources: CategorizedResources[T_ID, T_CogniteResourceList],
         loader: ResourceLoader,
         dry_run: bool,
     ) -> None:
         print_outs = []
         prefix = "Would have " if dry_run else ""
-        if to_create:
-            print_outs.append(f"{prefix}Created {_print_ids_or_length(loader.get_ids(to_create), limit=20)}")
-        if to_update:
-            print_outs.append(f"{prefix}Updated {_print_ids_or_length(loader.get_ids(to_update), limit=20)}")
-        if unchanged:
+        if resources.to_create:
+            print_outs.append(f"{prefix}Created {_print_ids_or_length(loader.get_ids(resources.to_create), limit=20)}")
+        if resources.to_update:
+            print_outs.append(f"{prefix}Updated {_print_ids_or_length(loader.get_ids(resources.to_update), limit=20)}")
+        if resources.unchanged:
             print_outs.append(
-                f"{'Untouched' if dry_run else 'Unchanged'} {_print_ids_or_length(loader.get_ids(unchanged), limit=5)}"
+                f"{'Untouched' if dry_run else 'Unchanged'} {_print_ids_or_length(loader.get_ids(resources.unchanged), limit=5)}"
             )
         prefix_message = f" {loader.display_name}: "
         if len(print_outs) == 1:

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py

@@ -46,6 +46,7 @@ from cognite_toolkit._cdf_tk.utils import (
     calculate_secure_hash,
 )
 from cognite_toolkit._cdf_tk.utils.cdf import read_auth, try_find_error
+from cognite_toolkit._cdf_tk.utils.text import suffix_description
 
 from .auth_loaders import GroupAllScopedLoader
 from .data_organization_loaders import DataSetsLoader
@@ -464,20 +465,15 @@ class FunctionScheduleLoader(
             )
             self.authentication_by_id[identifier] = credentials
             auth_hash = calculate_secure_hash(credentials.dump(camel_case=True), shorten=True)
-            extra_str = f" {self._hash_key}: {auth_hash}"
-            if "description" not in resource:
-                resource["description"] = extra_str[1:]
-            elif resource["description"].endswith(extra_str[1:]):
-                # The hash is already in the description
-                ...
-            elif len(resource["description"]) + len(extra_str) < self._description_character_limit:
-                resource["description"] += f"{extra_str}"
-            else:
-                LowSeverityWarning(f"Description is too long for schedule {identifier!r}. Truncating...").print_warning(
-                    console=self.console
-                )
-                truncation = self._description_character_limit - len(extra_str) - 3
-                resource["description"] = f"{resource['description'][:truncation]}...{extra_str}"
+            extra_str = f"{self._hash_key}: {auth_hash}"
+            resource["description"] = suffix_description(
+                extra_str,
+                resource.get("description"),
+                self._description_character_limit,
+                self.get_id(resource),
+                self.display_name,
+                self.console,
+            )
         return resources
 
     def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> FunctionScheduleWrite:

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/timeseries_loaders.py

@@ -1,5 +1,7 @@
+import json
 from collections.abc import Hashable, Iterable, Sequence
 from functools import lru_cache
+from pathlib import Path
 from typing import Any, cast, final
 
 from cognite.client.data_classes import (
@@ -28,7 +30,9 @@ from cognite_toolkit._cdf_tk.exceptions import (
 )
 from cognite_toolkit._cdf_tk.loaders._base_loaders import ResourceContainerLoader, ResourceLoader
 from cognite_toolkit._cdf_tk.resource_classes import TimeSeriesYAML
+from cognite_toolkit._cdf_tk.utils import calculate_hash
 from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_hashable, diff_list_identifiable, dm_identifier
+from cognite_toolkit._cdf_tk.utils.text import suffix_description
 
 from .auth_loaders import GroupAllScopedLoader, SecurityCategoryLoader
 from .classic_loaders import AssetLoader
@@ -216,6 +220,9 @@ class DatapointSubscriptionLoader(
         }
     )
 
+    _hash_key = "cdf-hash"
+    _description_character_limit = 1000
+
     @property
     def display_name(self) -> str:
         return "timeseries subscriptions"
@@ -325,25 +332,58 @@ class DatapointSubscriptionLoader(
     ) -> Iterable[DatapointSubscription]:
         return iter(self.client.time_series.subscriptions)
 
+    def load_resource_file(
+        self, filepath: Path, environment_variables: dict[str, str | None] | None = None
+    ) -> list[dict[str, Any]]:
+        resources = super().load_resource_file(filepath, environment_variables)
+        for resource in resources:
+            if "timeSeriesIds" not in resource and "instanceIds" not in resource:
+                continue
+            # If the timeSeriesIds or instanceIds is set, we need to add the auth hash to the description.
+            # such that we can detect if the subscription has changed.
+            content: dict[str, object] = {}
+            if "timeSeriesIds" in resource:
+                content["timeSeriesIds"] = resource["timeSeriesIds"]
+            if "instanceIds" in resource:
+                content["instanceIds"] = resource["instanceIds"]
+            timeseries_hash = calculate_hash(json.dumps(content), shorten=True)
+            extra_str = f"{self._hash_key}: {timeseries_hash}"
+            resource["description"] = suffix_description(
+                extra_str,
+                resource.get("description"),
+                self._description_character_limit,
+                self.get_id(resource),
+                self.display_name,
+                self.console,
+            )
+
+        return resources
+
     def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> DataPointSubscriptionWrite:
         if ds_external_id := resource.pop("dataSetExternalId", None):
             resource["dataSetId"] = self.client.lookup.data_sets.id(ds_external_id, is_dry_run)
         return DataPointSubscriptionWrite._load(resource)
 
     def dump_resource(self, resource: DatapointSubscription, local: dict[str, Any] | None = None) -> dict[str, Any]:
-        dumped = resource.as_write().dump()
+        if resource.filter is not None:
+            dumped = resource.as_write().dump()
+        else:
+            # If filter is not set, the subscription uses explicit timeSeriesIds, which are not returned in the
+            # response. Calling .as_write() in this case raises ValueError because either filter or
+            # timeSeriesIds must be set.
+            dumped = resource.dump()
+            for server_prop in ("createdTime", "lastUpdatedTime", "timeSeriesCount"):
+                dumped.pop(server_prop, None)
         local = local or {}
         if data_set_id := dumped.pop("dataSetId", None):
             dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id)
-        if "timeSeriesIds" not in dumped:
-            return dumped
-        # Sorting the timeSeriesIds in the local order
-        # Sorting in the same order as the local file.
-        ts_order_by_id = {ts_id: no for no, ts_id in enumerate(local.get("timeSeriesIds", []))}
-        end_of_list = len(ts_order_by_id)
-        dumped["timeSeriesIds"] = sorted(
-            dumped["timeSeriesIds"], key=lambda ts_id: ts_order_by_id.get(ts_id, end_of_list)
-        )
+        # timeSeriesIds and instanceIds are not returned in the response, so we need to add them
+        # to the dumped resource if they are set in the local resource. If there is a discrepancy between
+        # the local and dumped resource, th hash added to the description will change.
+        if "timeSeriesIds" in local:
+            dumped["timeSeriesIds"] = local["timeSeriesIds"]
+        if "instanceIds" in local:
+            dumped["instanceIds"] = local["instanceIds"]
         return dumped
 
     def diff_list(

cognite_toolkit/_cdf_tk/loaders/_worker.py

@@ -4,6 +4,7 @@ import re
 import warnings
 from collections.abc import Hashable
 from copy import deepcopy
+from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generic, cast
 
@@ -30,6 +31,14 @@ if TYPE_CHECKING:
     from cognite_toolkit._cdf_tk.data_classes._module_directories import ReadModule
 
 
+@dataclass
+class CategorizedResources(Generic[T_ID, T_CogniteResourceList]):
+    to_create: T_CogniteResourceList
+    to_update: T_CogniteResourceList
+    to_delete: list[T_ID]
+    unchanged: T_CogniteResourceList
+
+
 class ResourceWorker(
     Generic[T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList]
 ):
@@ -75,7 +84,7 @@ class ResourceWorker(
         is_dry_run: bool = False,
         force_update: bool = False,
         verbose: bool = False,
-    ) -> tuple[T_CogniteResourceList, T_CogniteResourceList, list[T_ID], T_CogniteResourceList]:
+    ) -> CategorizedResources:
         """Prepare resources for deployment by loading them from files, validating access, and categorizing them into create, update, delete, and unchanged lists.
 
         Args:
@@ -86,12 +95,7 @@ class ResourceWorker(
             verbose: Whether to print detailed information about the resources being processed.
 
         Returns:
-            A tuple containing:
-                - to_create: List of resources to create.
-                - to_update: List of resources to update.
-                - to_delete: List of resource IDs to delete.
-                - unchanged: List of resources that are unchanged.
-
+            CategorizedResources: A categorized list of resources to create, update, delete, and unchanged.
         """
         local_by_id = self.load_resources(filepaths, environment_variables, is_dry_run)
 
@@ -160,31 +164,28 @@ class ResourceWorker(
         cdf_resources: T_WritableCogniteResourceList,
         force_update: bool,
         verbose: bool,
-    ) -> tuple[T_CogniteResourceList, T_CogniteResourceList, list[T_ID], T_CogniteResourceList]:
-        to_create: T_CogniteResourceList
-        to_update: T_CogniteResourceList
-        to_delete: list[T_ID] = []
-        unchanged: T_CogniteResourceList
-        to_create, to_update, unchanged = (
-            self.loader.list_write_cls([]),
-            self.loader.list_write_cls([]),
-            self.loader.list_write_cls([]),
+    ) -> CategorizedResources:
+        resources: CategorizedResources[T_ID, T_CogniteResourceList] = CategorizedResources(
+            to_create=self.loader.list_write_cls([]),
+            to_update=self.loader.list_write_cls([]),
+            to_delete=[],
+            unchanged=self.loader.list_write_cls([]),
         )
         cdf_resource_by_id = {self.loader.get_id(resource): resource for resource in cdf_resources}
         for identifier, (local_dict, local_resource) in local_by_id.items():
             cdf_resource = cdf_resource_by_id.get(identifier)
             if cdf_resource is None:
-                to_create.append(local_resource)
+                resources.to_create.append(local_resource)
                 continue
             cdf_dict = self.loader.dump_resource(cdf_resource, local_dict)
             if not force_update and cdf_dict == local_dict:
-                unchanged.append(local_resource)
+                resources.unchanged.append(local_resource)
                 continue
             if self.loader.support_update:
-                to_update.append(local_resource)
+                resources.to_update.append(local_resource)
             else:
-                to_delete.append(identifier)
-                to_create.append(local_resource)
+                resources.to_delete.append(identifier)
+                resources.to_create.append(local_resource)
             if verbose:
                 diff_str = "\n".join(to_diff(cdf_dict, local_dict))
                 for sensitive in self.loader.sensitive_strings(local_resource):
@@ -196,4 +197,4 @@ class ResourceWorker(
                         expand=False,
                     )
                 )
-        return to_create, to_update, to_delete, unchanged
+        return resources

cognite_toolkit/_cdf_tk/resource_classes/workflow.py

@@ -0,0 +1,15 @@
+from cognite.client.data_classes import WorkflowUpsert
+from pydantic import Field
+
+from .base import ToolkitResource
+
+
+class WorkflowYAML(ToolkitResource):
+    _cdf_resource = WorkflowUpsert
+    external_id: str = Field(
+        max_length=255,
+        description="Identifier for a workflow. Must be unique for the project."
+        " No trailing or leading whitespace and no null characters allowed.",
+    )
+    description: str | None = Field(None, max_length=500)
+    data_set_external_id: str | None = None

cognite_toolkit/_cdf_tk/utils/text.py

@@ -0,0 +1,42 @@
+from collections.abc import Hashable
+
+from rich.console import Console
+
+from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning
+
+
+def suffix_description(
+    suffix: str,
+    description: str | None,
+    description_character_limit: int,
+    identifier: Hashable,
+    resource_type: str,
+    console: Console | None = None,
+) -> str:
+    """Appends a suffix to a description if it is not already present.
+    If the description is too long after appending the suffix, it will be truncated to fit within the character limit.
+
+    Args:
+        suffix: The suffix to append to the description.
+        description: The original description to which the suffix will be appended.
+        description_character_limit: The maximum number of characters allowed in the description after appending the suffix.
+        identifier: Hashable identifier for the resource, used in warnings.
+        resource_type:  Type of the resource, used in warnings.
+        console: Console object for printing warnings.
+
+    Returns:
+        str: The modified description with the suffix appended, or truncated if necessary.
+    """
+    if description is None or description == "":
+        return suffix
+    elif description.endswith(suffix):
+        # The suffix is already in the description
+        return description
+    elif len(description) + len(suffix) + 1 < description_character_limit:
+        return f"{description} {suffix}"
+    else:
+        LowSeverityWarning(f"Description is too long for {resource_type} {identifier!r}. Truncating...").print_warning(
+            console=console
+        )
+        truncation = description_character_limit - len(suffix) - 3
+        return f"{description[:truncation]}...{suffix}"

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.5.63
+      image: cognite/toolkit:0.5.65
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.5.63
+      image: cognite/toolkit:0.5.65
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.5.63"
+__version__ = "0.5.65"

{cognite_toolkit-0.5.63.dist-info → cognite_toolkit-0.5.65.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.5.63
+Version: 0.5.65
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.5.63.dist-info → cognite_toolkit-0.5.65.dist-info}/RECORD

@@ -1,10 +1,10 @@
 cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf.py,sha256=WWMslI-y2VbIYDMH19wnINebGwlOvAeYr-qkPRC1f68,5834
-cognite_toolkit/_version.py,sha256=NZyB_3d5nSrSdmUTR1qURqDAEzDzElqs5VAFoKfX_F8,23
+cognite_toolkit/_version.py,sha256=uPR7tuL4mJMppx_mxnT2t-XCK6LTKMl7XrNVV0IoCLM,23
 cognite_toolkit/config.dev.yaml,sha256=CIDmi1OGNOJ-70h2BNCozZRmhvU5BfpZoh6Q04b8iMs,109
 cognite_toolkit/_builtin_modules/README.md,sha256=roU3G05E6ogP5yhw4hdIvVDKV831zCh2pzt9BVddtBg,307
 cognite_toolkit/_builtin_modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cognite_toolkit/_builtin_modules/cdf.toml,sha256=kjIlSaM6fJ5C09gRiWydBtFY7C2IuoHBUMue1zv7y_I,273
+cognite_toolkit/_builtin_modules/cdf.toml,sha256=LGJ7fwBn4yuwF9I5hjQeBFhQ-B6cPwxxW_sUsYKBPJw,273
 cognite_toolkit/_builtin_modules/packages.toml,sha256=RdY44Sxvh6sUtAkgp1dHID1mtqkOTzP_rbZL2Q27fYw,1147
 cognite_toolkit/_builtin_modules/bootcamp/README.md,sha256=iTVqoy3PLpC-xPi5pbuMIAEHILBSfWTGLexwa1AltpY,211
 cognite_toolkit/_builtin_modules/bootcamp/default.config.yaml,sha256=MqYTcRiz03bow4LT8E3jumnd_BsqC5SvjgYOVVkHGE0,93
@@ -499,7 +499,7 @@ cognite_toolkit/_cdf_tk/apps/_auth_app.py,sha256=ER7uYb3ViwsHMXiQEZpyhwU6TIjKaB9
 cognite_toolkit/_cdf_tk/apps/_core_app.py,sha256=-4ABeNtC0cxw7XvCRouPzTvlmqsS0NRR-jLgMGadW2I,13712
 cognite_toolkit/_cdf_tk/apps/_dump_app.py,sha256=UXmB8oFwVLOmxJBlxxLIBMLPCLwdgyaFfuG6Ex-GZh4,25608
 cognite_toolkit/_cdf_tk/apps/_landing_app.py,sha256=v4t2ryxzFre7y9IkEPIDwmyJDO8VDIIv6hIcft5TjpQ,422
-cognite_toolkit/_cdf_tk/apps/_migrate_app.py,sha256=GRsOlqYAWB0rsZsdTJTGfjPm1OkbUq7xBrM4pzQRKoY,3708
+cognite_toolkit/_cdf_tk/apps/_migrate_app.py,sha256=ifzl63MBjxV3gdf3h7dztGLf8VbuXHeRnWG-p4gBCGE,5299
 cognite_toolkit/_cdf_tk/apps/_modules_app.py,sha256=tjCP-QbuPYd7iw6dkxnhrrWf514Lr25_oVgSJyJcaL8,6642
 cognite_toolkit/_cdf_tk/apps/_populate_app.py,sha256=PGUqK_USOqdPCDvUJI-4ne9TN6EssC33pUbEeCmiLPg,2805
 cognite_toolkit/_cdf_tk/apps/_profile_app.py,sha256=TaKTOgkd538QyIWBRdAILJ-TotBxYreZgWBqK4yrebQ,2562
@@ -524,7 +524,7 @@ cognite_toolkit/_cdf_tk/client/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
 cognite_toolkit/_cdf_tk/client/api/dml.py,sha256=8b1lo86JdvfEsz9mP2rx0Mp9fyWsU6mbXHqLBtvSidU,3546
 cognite_toolkit/_cdf_tk/client/api/extended_data_modeling.py,sha256=V9a-Ep_xlxd9KQN1D15GOLmg6KrSADciPWlmgAJuaBY,10481
 cognite_toolkit/_cdf_tk/client/api/extended_raw.py,sha256=9DVbM2aWmIyzbaW-lh10_pzVYJUEQFnIKnxvt413Bjk,2118
-cognite_toolkit/_cdf_tk/client/api/extended_timeseries.py,sha256=8GsMDq3F9tb_bR9c6yViRErjf0AsUJZbrZx9JTQZvXY,15150
+cognite_toolkit/_cdf_tk/client/api/extended_timeseries.py,sha256=wi4EzOa6Pup_YKcQ3b3SMvKminJ1kee9-dgYOLwqSQc,17804
 cognite_toolkit/_cdf_tk/client/api/fixed_transformations.py,sha256=MEC25h_J_fm3wHa-rVJi2V_b5D1Fn3wNapCmIGU2faU,5625
 cognite_toolkit/_cdf_tk/client/api/location_filters.py,sha256=kUe58xzt2iYdztleep8ocsdE17lOWgqbPjELJnfhoHU,3188
 cognite_toolkit/_cdf_tk/client/api/lookup.py,sha256=E4YEkk8TzpWQ2v_NacISgXyA4xoelmayqSwaq4vykdk,12250
@@ -562,7 +562,7 @@ cognite_toolkit/_cdf_tk/client/data_classes/streamlit_.py,sha256=OGoMQ_K88F9vSZu
 cognite_toolkit/_cdf_tk/client/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/client/utils/_concurrency.py,sha256=z6gqFv-kw80DsEpbaR7sI0-_WvZdOdAsR4VoFvTqvyU,1309
 cognite_toolkit/_cdf_tk/client/utils/_http_client.py,sha256=oXNKrIaizG4WiSAhL_kSCHAuL4aaaEhCU4pOJGxh6Xs,483
-cognite_toolkit/_cdf_tk/commands/__init__.py,sha256=6rUv97s6CB5Fje6eg2X3cd9Za9rYJY51xUcPk_RoJT8,1261
+cognite_toolkit/_cdf_tk/commands/__init__.py,sha256=2oQWO2QuSBsXtaLjjKrbmpH909R6gUlVfL55GuyyHwY,1311
 cognite_toolkit/_cdf_tk/commands/_base.py,sha256=3Zc3ffR8mjZ1eV7WrC-Y1sYmyMzdbbJDDmsiKEMEJwo,2480
 cognite_toolkit/_cdf_tk/commands/_changes.py,sha256=3bR_C8p02IW6apexwAAoXuneBM4RcUGdX6Hw_Rtx7Kg,24775
 cognite_toolkit/_cdf_tk/commands/_cli_commands.py,sha256=6nezoDrw3AkF8hANHjUILgTj_nbdzgT0siweaKI35Fk,1047
@@ -575,7 +575,7 @@ cognite_toolkit/_cdf_tk/commands/auth.py,sha256=T6hb90PnlRiTkhihEUvLCbNFyt1_4ML3
 cognite_toolkit/_cdf_tk/commands/build_cmd.py,sha256=Za0hYNlSE8yMuAdczKYtTgvcqE6DIbCGve1E2OfXekI,30474
 cognite_toolkit/_cdf_tk/commands/clean.py,sha256=qKHrhkjzerC-oQgkZ_61n7vZw11fxLJsJHILyd9Z5UQ,14418
 cognite_toolkit/_cdf_tk/commands/collect.py,sha256=zBMKhhvjOpuASMnwP0eeHRI02tANcvFEZgv0CQO1ECc,627
-cognite_toolkit/_cdf_tk/commands/deploy.py,sha256=g_mceI5Hq55uz9p5WC7yPVNPR6MBhIMTcBt9G_cWkJQ,19208
+cognite_toolkit/_cdf_tk/commands/deploy.py,sha256=hXviaYPJdx4vxkNb-KbgHydBznfc9LuHxUyW-3M21qI,19218
 cognite_toolkit/_cdf_tk/commands/dump_data.py,sha256=U_e-fEAEphpkJMlDTHQvQ1F0k3qEMvd0m7zc20XvcQY,21668
 cognite_toolkit/_cdf_tk/commands/dump_resource.py,sha256=Dt8jlkmtpRtzPDMEjKdpOJPFr92k7Mw-BWkRsE9CJ8s,20515
 cognite_toolkit/_cdf_tk/commands/featureflag.py,sha256=VPz7FrjVQFqjkz8BYTP2Np3k7BTLFMq_eooNSqmb2ms,1034
@@ -584,10 +584,11 @@ cognite_toolkit/_cdf_tk/commands/modules.py,sha256=lYImbi7eX07j2lbE_8xJ5uix9xa2l
 cognite_toolkit/_cdf_tk/commands/pull.py,sha256=t7KQCxpoFDNBWTYPohK7chrRzPyAOGVmfaY7iBLnTqM,39286
 cognite_toolkit/_cdf_tk/commands/repo.py,sha256=vQfLMTzSnI4w6eYCQuMnZ_xXVAVjyLnST4Tmu2zgNfE,3874
 cognite_toolkit/_cdf_tk/commands/run.py,sha256=88AkfCdS4gXHA4I5ZhdU3HWWA5reOTGbfaauM-Yvp8o,37407
-cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py,sha256=jV7zzYhBxQWGpAWCu-dJ9QxUHp3DBwz2KEQihGb2UuI,161
-cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py,sha256=uONZwIkdD7pcllzu4VT0gHar5x5qpdMayAaQFI4kozQ,6338
-cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py,sha256=y7Fc_bZvavcNTS9LxwiiMnYIDkGOmMXpbm8hTnAjkYw,2593
-cognite_toolkit/_cdf_tk/commands/_migrate/prepare.py,sha256=65oSmngUIDMGzKmwXDLft1ITPiA8COEuIcTnTafo5cg,2511
+cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py,sha256=E36hrJ71Wfm9y8k3qLXTo-cy2SCxtSy9bH32MixMtm8,226
+cognite_toolkit/_cdf_tk/commands/_migrate/assets.py,sha256=o9kbOLn0WaptXBpLWmlndBY5Wu5AeQsnUvV_lGBzcqY,9296
+cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py,sha256=tj0IwrW8Tk0U_1ZCte5pbC84Lt95DLe6aVWdhLaI7EA,6776
+cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py,sha256=pmpHqcv5bOW_lYhG-JfqfesTiMWGSH8dXzEVPbSkDXk,2633
+cognite_toolkit/_cdf_tk/commands/_migrate/prepare.py,sha256=vtJjmNEVDiYTNajRry1oCqG_yBlHqq00M_NuInvQGZ0,2271
 cognite_toolkit/_cdf_tk/commands/_migrate/timeseries.py,sha256=noPHSkKKaziI3X0KZLpZY_xjC5yejRX2d9TJgTtyZCo,9659
 cognite_toolkit/_cdf_tk/data_classes/__init__.py,sha256=Z7ODYLcqrRpo0Cmfx79DDhsA6eEK4hvNST_Qko1vRv0,1645
 cognite_toolkit/_cdf_tk/data_classes/_base.py,sha256=qyXObVP1SX5Lzqy8cYBZssV9NL3v0Q0-y-pLNF1Ok1I,2652
@@ -605,7 +606,7 @@ cognite_toolkit/_cdf_tk/data_classes/_yaml_comments.py,sha256=zfuDu9aAsb1ExeZBAJ
 cognite_toolkit/_cdf_tk/loaders/__init__.py,sha256=9giALvw48KIry7WWdCUxA1AvlVFCAR0bOJ5tKAhy-Lk,6241
 cognite_toolkit/_cdf_tk/loaders/_base_loaders.py,sha256=sF9D7ImyHmjbLBGVM66D2xSmOj8XnG3LmDqlQQZRarQ,20502
 cognite_toolkit/_cdf_tk/loaders/_data_loaders.py,sha256=GHFylB-LwpYdOHI_hwWPL68TMO3D99iFOzT-oAPZFLc,9190
-cognite_toolkit/_cdf_tk/loaders/_worker.py,sha256=xenPKmnx2xbwIzm-KVwSjMTxWoni1_NO4xz7mfDs-EY,9472
+cognite_toolkit/_cdf_tk/loaders/_worker.py,sha256=E1y_xFzsb3mSZiSJJggvRLB6tT-0xBoKlqpa7UaX5us,9385
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/__init__.py,sha256=d8ucrEuVA8W9zVon8X6KvRejdDABWNCc6Qco-9BR9AQ,2964
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/agent_loaders.py,sha256=2hsaMB9lyNPAWUGKQ3EMA46ceUmbNefdoBx3huWF4fo,3128
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/auth_loaders.py,sha256=1wfR9yV_htDBmG6DATm5eN0m0r9wuHg4H0L8X9bAp20,26332
@@ -615,7 +616,7 @@ cognite_toolkit/_cdf_tk/loaders/_resource_loaders/datamodel_loaders.py,sha256=xK
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/extraction_pipeline_loaders.py,sha256=zqNPiIX1xvYV8alpxPKMqyy4QlH6oqDYNtrITC7ZKdo,18188
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/fieldops_loaders.py,sha256=pzjOjnxaJRXId-b0GZ31a6Mab8BdNnKXx3zUiBaxB0E,12203
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/file_loader.py,sha256=49uHmkYA5dzL7fyXYRIeZCkYnB5laclO-6Pis_qhVes,17633
-cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py,sha256=tO5i_xf1MBt_eQ6VUFJi6VZvusq16Xr33zqirp-Jers,26356
+cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py,sha256=5_p3G0Kuz08saJs_4T6hD26BwHsVvvFG20IhAPZlZbw,25937
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/group_scoped_loader.py,sha256=Rerw0Y6tY6Nle3vmyl4nhX5lRsVkUVcnp39qZ3R_tDs,1830
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/hosted_extractors.py,sha256=lZuq_dj7_r0O3BcM6xnAenAXAtcJ3ke_LO1IguRKCiE,17527
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/industrial_tool_loaders.py,sha256=j1FsynrFk_iPIAsHuqjXmtbMM7ZM6RVlpQkxCH9ATO0,8493
@@ -624,7 +625,7 @@ cognite_toolkit/_cdf_tk/loaders/_resource_loaders/raw_loaders.py,sha256=RM-zxDX6
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/relationship_loader.py,sha256=KRWGHk11vRWhFiQUF0vChv95ElGY-E_VGQNCHkuY5Y4,7219
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/robotics_loaders.py,sha256=tgPdVA4B8gNfHeHlEMzDgBaWs2jj-FSZAm1O1r5Uzrc,17224
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/three_d_model_loaders.py,sha256=IcnZ9wyW6Dpl83nlOdAM44UyrMfgum2hLdqi1u2AEuw,8288
-cognite_toolkit/_cdf_tk/loaders/_resource_loaders/timeseries_loaders.py,sha256=Xy6Wu4JU4kDVCbiVV3Y0GXBcix6Y-snLDwtj6g33vow,15281
+cognite_toolkit/_cdf_tk/loaders/_resource_loaders/timeseries_loaders.py,sha256=PU2gRi-9EQKooZ8BLpKnQOtg3lNQ0lRBW0C26BU8r_0,17288
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/transformation_loaders.py,sha256=D0iD7jWPetiUM35JKQ1M_By086k4n5co9RaA8-VfF_A,35304
 cognite_toolkit/_cdf_tk/loaders/_resource_loaders/workflow_loaders.py,sha256=KmqDOIg6ttVqL_7widJgW5rNjs4eipd28AcLDs_bwDc,28427
 cognite_toolkit/_cdf_tk/prototypes/import_app.py,sha256=7dy852cBlHI2RQF1MidSmxl0jPBxekGWXnd2VtI7QFI,1899
@@ -671,6 +672,7 @@ cognite_toolkit/_cdf_tk/resource_classes/timeseries.py,sha256=wVlXR6tsmC-xmCnF4d
 cognite_toolkit/_cdf_tk/resource_classes/transformation_destination.py,sha256=AjfGS0JTGGG7aqUiezxtaiv_dBp-QMvHLwulXUIW8sA,6525
 cognite_toolkit/_cdf_tk/resource_classes/transformation_schedule.py,sha256=eTU1pEtR9z2SGeZU4AcXAGNWGfqf1v3L-wPDEVCSD9s,406
 cognite_toolkit/_cdf_tk/resource_classes/transformations.py,sha256=dt1coxSflgb7-NGHomYt8jFHPZni-xHgbCzU8i5B7-Q,3555
+cognite_toolkit/_cdf_tk/resource_classes/workflow.py,sha256=fMNfW93D8tdVwO7YgEYYiYvpktSMx4i0viIFg0gD2VY,512
 cognite_toolkit/_cdf_tk/tk_warnings/__init__.py,sha256=DmvCZhG59NZ7yepDm4s-gy92ysQBJ2t-LnlsLatNwOw,2262
 cognite_toolkit/_cdf_tk/tk_warnings/base.py,sha256=RMtJ_0iE1xfbTKkvWLkAo5AgscjkNDpT0lmDZ3I8aeI,4435
 cognite_toolkit/_cdf_tk/tk_warnings/fileread.py,sha256=9xE8i8F_HhrRaQi6IGcE240j4TRZQzrOd7gMByjsXEk,8924
@@ -693,17 +695,18 @@ cognite_toolkit/_cdf_tk/utils/sentry_utils.py,sha256=YWQdsePeFpT214-T-tZ8kEsUyC8
 cognite_toolkit/_cdf_tk/utils/sql_parser.py,sha256=RhUPWjVjwb9RBv1fixmG7bKvAb4JT_CC0O7Aqnx5Pgg,6196
 cognite_toolkit/_cdf_tk/utils/table_writers.py,sha256=wEBVlfCFv5bLLy836UiXQubwSxo8kUlSFZeQxnHrTX4,17932
 cognite_toolkit/_cdf_tk/utils/tarjan.py,sha256=mr3gMzlrkDadn1v7u7-Uzao81KKiM3xfXlZ185HL__A,1359
+cognite_toolkit/_cdf_tk/utils/text.py,sha256=gBl3o60dXRlEBsg8izdnOmuLo86jr35pQFZcxnKdNSY,1715
 cognite_toolkit/_repo_files/.env.tmpl,sha256=UmgKZVvIp-OzD8oOcYuwb_6c7vSJsqkLhuFaiVgK7RI,972
 cognite_toolkit/_repo_files/.gitignore,sha256=3exydcQPCJTldGFJoZy1RPHc1horbAprAoaShU8sYnM,5262
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=KVBxW8urCRDtVlJ6HN-kYmw0NCpW6c4lD-nlxz9tZsQ,692
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=Cp4KYraeWPjP8SnnEIbJoJnjmrRUwc982DPjOOzy2iM,722
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=kPSbNvw3tI5xsGHKjUDkCMebY6HuTTMwJk7Q3yR7EB4,667
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=_5mG5DphhCXHAScbPTnIaTkJ9tYTSKtyXVlO6aeQyQo,2430
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=r9DQ1_--OcUVFusWhTykM5NKmEn17ijBDatY5XNAoQ8,667
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=Go3m_277I0Kv2kuBFjPhYQAf0defj2ORfStEchgy7aA,2430
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=63nWYI_MHU5EuPwEX_inEAQxxiD5P6k8IAmlgl4CxpE,8082
-cognite_toolkit-0.5.63.dist-info/METADATA,sha256=o1oTTOVyWL8Lzlm-3ho6SMvBuN57paaH0UJJmfdRSeQ,4410
-cognite_toolkit-0.5.63.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_toolkit-0.5.63.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
-cognite_toolkit-0.5.63.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
-cognite_toolkit-0.5.63.dist-info/RECORD,,
+cognite_toolkit-0.5.65.dist-info/METADATA,sha256=J_4JmnX17kzzGuODbN5Ght5chcP3ZMZPtwKquCTxfUo,4410
+cognite_toolkit-0.5.65.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_toolkit-0.5.65.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
+cognite_toolkit-0.5.65.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
+cognite_toolkit-0.5.65.dist-info/RECORD,,