cognite-toolkit 0.5.97__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/_migrate/base.py

@@ -1,88 +0,0 @@
-from abc import ABC, abstractmethod
-
-from cognite.client.data_classes.capabilities import (
-    Capability,
-    DataModelInstancesAcl,
-    DataModelsAcl,
-    SpaceIDScope,
-)
-
-from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
-from cognite_toolkit._cdf_tk.constants import DMS_INSTANCE_LIMIT_MARGIN
-from cognite_toolkit._cdf_tk.exceptions import (
-    AuthenticationError,
-    ToolkitMigrationError,
-    ToolkitValueError,
-)
-from cognite_toolkit._cdf_tk.utils import humanize_collection
-
-from .data_model import INSTANCE_SOURCE_VIEW_ID
-
-
-class BaseMigrateCommand(ToolkitCommand, ABC):
-    @property
-    @abstractmethod
-    def schema_spaces(self) -> list[str]:
-        """Return the schema spaces used by this migration command."""
-        raise NotImplementedError()
-
-    def source_acl(self, data_set_id: list[int]) -> Capability | None:
-        """Return the source ACL for the given data set IDs."""
-        # This method should be implemented in subclasses that needs access to a specific source ACL.
-        # such as TimeSeries, Files, Assets, and so on.
-        return None
-
-    def validate_access(
-        self, client: ToolkitClient, instance_spaces: list[str], data_set_ids: list[int] | None = None
-    ) -> None:
-        required_capabilities: list[Capability] = [
-            DataModelsAcl(actions=[DataModelsAcl.Action.Read], scope=SpaceIDScope(self.schema_spaces)),
-            DataModelInstancesAcl(
-                actions=[
-                    DataModelInstancesAcl.Action.Read,
-                    DataModelInstancesAcl.Action.Write,
-                    DataModelInstancesAcl.Action.Write_Properties,
-                ],
-                scope=SpaceIDScope(instance_spaces),
-            ),
-        ]
-        if data_set_ids is not None:
-            source_acl = self.source_acl(data_set_ids)
-            if source_acl is None:
-                raise ValueError(
-                    "Bug in Toolkit: the source ACL is not defined for this migration command. "
-                    "Please implement the source_acl method."
-                )
-            required_capabilities.append(source_acl)
-        if missing := client.iam.verify_capabilities(required_capabilities):
-            raise AuthenticationError(f"Missing required capabilities: {humanize_collection(missing)}.", missing)
-
-    @staticmethod
-    def validate_instance_source_exists(client: ToolkitClient) -> None:
-        view = client.data_modeling.views.retrieve(INSTANCE_SOURCE_VIEW_ID)
-        if not view:
-            raise ToolkitMigrationError(
-                f"The migration mapping view {INSTANCE_SOURCE_VIEW_ID} does not exist. "
-                f"Please run the `cdf migrate prepare` command to deploy the migration data model."
-            )
-
-    def validate_available_capacity(self, client: ToolkitClient, instance_count: int) -> None:
-        """Validate that the project has enough capacity to accommodate the migration."""
-
-        stats = client.data_modeling.statistics.project()
-
-        available_capacity = stats.instances.instances_limit - stats.instances.instances
-        available_capacity_after = available_capacity - instance_count
-
-        if available_capacity_after < DMS_INSTANCE_LIMIT_MARGIN:
-            raise ToolkitValueError(
-                "Cannot proceed with migration, not enough instance capacity available. Total capacity after migration"
-                f" would be {available_capacity_after:,} instances, which is less than the required margin of"
-                f" {DMS_INSTANCE_LIMIT_MARGIN:,} instances. Please increase the instance capacity in your CDF project"
-                f" or delete some existing instances before proceeding with the migration of {instance_count:,} assets."
-            )
-        total_instances = stats.instances.instances + instance_count
-        self.console(
-            f"Project has enough capacity for migration. Total instances after migration: {total_instances:,}."
-        )

cognite_toolkit/_cdf_tk/commands/_migrate/canvas.py

@@ -1,145 +0,0 @@
-from uuid import uuid4
-
-from cognite.client.exceptions import CogniteException
-
-from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.canvas import (
-    CANVAS_INSTANCE_SPACE,
-    Canvas,
-    ContainerReferenceApply,
-    FdmInstanceContainerReferenceApply,
-)
-from cognite_toolkit._cdf_tk.client.data_classes.migration import InstanceSource
-from cognite_toolkit._cdf_tk.exceptions import ToolkitMigrationError
-from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, LowSeverityWarning, MediumSeverityWarning
-from cognite_toolkit._cdf_tk.utils import humanize_collection
-from cognite_toolkit._cdf_tk.utils.interactive_select import InteractiveCanvasSelect
-
-from .base import BaseMigrateCommand
-from .data_model import INSTANCE_SOURCE_VIEW_ID
-
-
-class MigrationCanvasCommand(BaseMigrateCommand):
-    canvas_schema_space = Canvas.get_source().space
-    # Note sequences are not supported in Canvas, so we do not include them here.
-    asset_centric_resource_types = frozenset({"asset", "event", "file", "timeseries"})
-
-    @property
-    def schema_spaces(self) -> list[str]:
-        return [self.canvas_schema_space, INSTANCE_SOURCE_VIEW_ID.space]
-
-    def migrate_canvas(
-        self,
-        client: ToolkitClient,
-        external_ids: list[str] | None = None,
-        dry_run: bool = False,
-        verbose: bool = False,
-    ) -> None:
-        self.validate_access(client, [CANVAS_INSTANCE_SPACE])
-        self.validate_instance_source_exists(client)
-        external_ids = external_ids or InteractiveCanvasSelect(client).select_external_ids()
-        if external_ids is None or not external_ids:
-            self.console("No canvases selected for migration.")
-            return
-        action = "Would migrate" if dry_run else "Migrating"
-        self.console(f"{action} {len(external_ids)} canvases.")
-        for external_id in external_ids:
-            self._migrate_single_canvas(client, external_id, dry_run=dry_run, verbose=verbose)
-
-    def _migrate_single_canvas(
-        self,
-        client: ToolkitClient,
-        external_id: str,
-        dry_run: bool = False,
-        verbose: bool = False,
-    ) -> None:
-        canvas = client.canvas.industrial.retrieve(external_id=external_id)
-        if canvas is None:
-            self.warn(MediumSeverityWarning(f"Canvas with external ID '{external_id}' not found. Skipping.. "))
-            return
-        update = canvas.as_write()
-        to_migrate = [
-            ref
-            for ref in update.container_references
-            if ref.container_reference_type in self.asset_centric_resource_types
-        ]
-        if not to_migrate:
-            self.warn(
-                LowSeverityWarning(
-                    f"Canvas with name '{canvas.canvas.name}' does not have any asset-centric references. Skipping.. "
-                )
-            )
-        if verbose:
-            self.console(f"Found canvas: {canvas.canvas.name}")
-        reference_ids = [ref.as_asset_centric_id() for ref in to_migrate]
-        instance_sources = client.migration.instance_source.retrieve(reference_ids)
-        source_by_reference_id = {source.as_asset_centric_id(): source for source in instance_sources}
-        missing = set(reference_ids) - set(source_by_reference_id.keys())
-        if missing:
-            self.warn(
-                HighSeverityWarning(
-                    f"Canvas '{canvas.canvas.name}' has references to resources that are not been migrated: {humanize_collection(missing)}. Skipping.. "
-                )
-            )
-            return
-        if dry_run:
-            self.console(
-                f"Canvas '{canvas.canvas.name}' is ready for migration all {len(instance_sources)} references asset-centric resources found."
-            )
-            return
-        if verbose:
-            self.console(
-                f"Migrating canvas '{canvas.canvas.name}' with {len(instance_sources)} references to asset-centric resources."
-            )
-        backup = canvas.as_write().create_backup()
-
-        update.container_references = [
-            ref
-            for ref in update.container_references
-            if ref.container_reference_type not in self.asset_centric_resource_types
-        ]
-        for ref in to_migrate:
-            source = source_by_reference_id[ref.as_asset_centric_id()]
-            fdm_ref = self.migrate_container_reference(ref, source, canvas.canvas.external_id)
-            update.fdm_instance_container_references.append(fdm_ref)
-
-        try:
-            client.canvas.industrial.create(backup)
-        except CogniteException as e:
-            raise ToolkitMigrationError(f"Failed to create backup for canvas '{canvas.canvas.name}': {e!s}. ") from e
-        try:
-            client.canvas.industrial.update(update)
-        except CogniteException as e:
-            raise ToolkitMigrationError(
-                f"Failed to migrate canvas '{canvas.canvas.name}': {e!s}. A backup was created with external ID '{backup.canvas.external_id}'."
-            ) from e
-        else:
-            self.console(
-                f'Canvas "{canvas.canvas.name}" migrated successfully with {len(to_migrate)} references to data model instances.'
-            )
-
-    @classmethod
-    def migrate_container_reference(
-        cls, reference: ContainerReferenceApply, source: InstanceSource, canvas_external_id: str
-    ) -> FdmInstanceContainerReferenceApply:
-        """Migrate a single container reference by replacing the asset-centric ID with the data model instance ID."""
-        consumer_view = source.consumer_view()
-        new_external_id = f"{canvas_external_id}_{uuid4()}"
-        return FdmInstanceContainerReferenceApply(
-            external_id=new_external_id,
-            id_=reference.id_,
-            container_reference_type="fdmInstance",
-            instance_space=source.space,
-            instance_external_id=source.external_id,
-            view_space=consumer_view.space,
-            view_external_id=consumer_view.external_id,
-            view_version=consumer_view.version,
-            label=reference.label,
-            properties_=reference.properties_,
-            x=reference.x,
-            y=reference.y,
-            width=reference.width,
-            height=reference.height,
-            max_width=reference.max_width,
-            max_height=reference.max_height,
-        )

cognite_toolkit/_cdf_tk/commands/_migrate/files.py

@@ -1,165 +0,0 @@
-from pathlib import Path
-
-import questionary
-from cognite.client.data_classes.capabilities import (
-    Capability,
-    DataSetScope,
-    FilesAcl,
-)
-from cognite.client.data_classes.data_modeling import ViewId
-from cognite.client.data_classes.data_modeling.cdm.v1 import CogniteFileApply
-from cognite.client.exceptions import CogniteAPIError
-from rich import print
-from rich.panel import Panel
-from rich.progress import track
-
-from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.extended_filemetdata import ExtendedFileMetadata
-from cognite_toolkit._cdf_tk.exceptions import (
-    AuthenticationError,
-    ToolkitValueError,
-)
-from cognite_toolkit._cdf_tk.utils import humanize_collection
-from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
-
-from .base import BaseMigrateCommand
-from .data_classes import MigrationMappingList
-
-
-class MigrateFilesCommand(BaseMigrateCommand):
-    cdf_cdm = "cdf_cdm"
-    view_id = ViewId(cdf_cdm, "CogniteFile", "v1")
-    chunk_size = 1000
-
-    @property
-    def schema_spaces(self) -> list[str]:
-        return [self.cdf_cdm]
-
-    def source_acl(self, data_set_ids: list[int]) -> Capability:
-        return FilesAcl(
-            actions=[FilesAcl.Action.Read, FilesAcl.Action.Write],
-            scope=DataSetScope(data_set_ids),
-        )
-
-    def migrate_files(
-        self,
-        client: ToolkitClient,
-        mapping_file: Path,
-        dry_run: bool = False,
-        verbose: bool = False,
-        auto_yes: bool = False,
-    ) -> None:
-        """Migrate resources from Asset-Centric to data modeling in CDF."""
-        mappings = MigrationMappingList.read_mapping_file(mapping_file)
-        self.validate_access(client, list(mappings.spaces()), list(mappings.get_data_set_ids()))
-        self._validate_files(client, mappings)
-        self.validate_available_capacity(client, len(mappings))
-
-        if dry_run:
-            self.console(f"Dry run mode. Would have migrated {len(mappings):,} Files to CogniteFiles.")
-            return
-        if not auto_yes and self._confirm(mappings) is False:
-            return
-        self._migrate(client, mappings, verbose)
-
-    def _validate_files(self, client: ToolkitClient, mappings: MigrationMappingList) -> None:
-        total_validated = 0
-        chunk: MigrationMappingList
-        for chunk in track(  # type: ignore[assignment]
-            chunker_sequence(mappings, size=self.chunk_size),
-            description="Validating...",
-            total=len(mappings) // self.chunk_size + 1,
-        ):
-            try:
-                files = client.files.retrieve_multiple(
-                    ids=chunk.get_ids(),
-                    external_ids=chunk.get_external_ids(),
-                    ignore_unknown_ids=True,
-                )
-            except CogniteAPIError as e:
-                raise AuthenticationError(
-                    f"Failed to retrieve Files. This is likely due to lack of permissions: {e!s}"
-                ) from e
-
-            missing_count = len(files) - len(mappings)
-            if missing_count > 0:
-                raise ToolkitValueError(f"{missing_count} Files are missing in CDF.")
-
-            missing_file_content = [file for file in files if file.uploaded is not True]
-            if missing_file_content:
-                raise ToolkitValueError(
-                    f"The following files does not have file content yet: {humanize_collection(missing_file_content)}. "
-                    "You can only migrate files that have file content uploaded."
-                )
-
-            existing_result = client.data_modeling.instances.retrieve(chunk.as_node_ids())
-            if len(existing_result.nodes) != 0:
-                raise ToolkitValueError(
-                    "Some of the Files you are trying to migrate already exist in Data Modeling. "
-                    f"Please remove the following files from the mapping file {humanize_collection(existing_result.nodes.as_ids())}"
-                )
-            total_validated += len(files)
-        print(
-            f"Validated {total_validated:,} Files for migration. "
-            f"{len(mappings):,} mappings provided in the mapping file."
-        )
-
-    @staticmethod
-    def _confirm(mappings: MigrationMappingList) -> bool:
-        print(
-            Panel(
-                f"[red]WARNING:[/red] This operation [bold]cannot be undone[/bold]! "
-                f"{len(mappings):,} Files will linked to the new CogniteFiles. "
-                "This linking cannot be undone",
-                style="bold",
-                title="Migrate asset-centric Files to CogniteFiles",
-                title_align="left",
-                border_style="red",
-                expand=False,
-            )
-        )
-
-        if not questionary.confirm("Are you really sure you want to continue?", default=False).ask():
-            print("Migration cancelled by user.")
-            return False
-        return True
-
-    def _migrate(self, client: ToolkitClient, mappings: MigrationMappingList, verbose: bool) -> None:
-        print("Migrating Files to CogniteFiles...")
-        total_migrated = 0
-        for chunk in track(
-            chunker_sequence(mappings, size=self.chunk_size),
-            description="Migrating Files to CogniteFiles...",
-            total=len(mappings) // self.chunk_size + 1,
-        ):  # type: ignore[assignment]
-            if verbose:
-                print(f"Migrating {len(chunk):,} Files...")
-
-            # Set pending IDs for the chunk of mappings
-            try:
-                pending_files = client.files.set_pending_ids(chunk.as_pending_ids())
-            except CogniteAPIError as e:
-                raise ToolkitValueError(f"Failed to set pending IDs for Files: {e!s}") from e
-
-            # The ExtendedTimeSeriesList is iterating ExtendedTimeSeries objects.
-            converted_files = [self.as_cognite_file(file) for file in pending_files]  # type: ignore[arg-type]
-            try:
-                created = client.data_modeling.instances.apply_fast(converted_files)
-            except CogniteAPIError as e:
-                raise ToolkitValueError(f"Failed to apply Files: {e!s}") from e
-            if verbose:
-                print(f"Created {len(created):,} CogniteFiles.")
-            total_migrated += len(created)
-        print(f"Successfully migrated {total_migrated:,} Files to CogniteFiles.")
-
-    @classmethod
-    def as_cognite_file(cls, file: ExtendedFileMetadata) -> CogniteFileApply:
-        if file.pending_instance_id is None:
-            raise ToolkitValueError("ExtendedFiles must have a pending_instance_id set before migration.")
-
-        return CogniteFileApply(
-            space=file.pending_instance_id.space,
-            external_id=file.pending_instance_id.external_id,
-            name=file.name,
-            mime_type=file.mime_type,
-        )

cognite_toolkit/_cdf_tk/commands/_migrate/timeseries.py

@@ -1,165 +0,0 @@
-from pathlib import Path
-
-import questionary
-from cognite.client.data_classes.capabilities import (
-    Capability,
-    DataSetScope,
-    TimeSeriesAcl,
-)
-from cognite.client.data_classes.data_modeling import DirectRelationReference, ViewId
-from cognite.client.data_classes.data_modeling.cdm.v1 import CogniteTimeSeriesApply
-from cognite.client.exceptions import CogniteAPIError
-from rich import print
-from rich.panel import Panel
-from rich.progress import track
-
-from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.extended_timeseries import ExtendedTimeSeries
-from cognite_toolkit._cdf_tk.exceptions import (
-    AuthenticationError,
-    ToolkitValueError,
-)
-from cognite_toolkit._cdf_tk.utils import humanize_collection
-from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
-
-from .base import BaseMigrateCommand
-from .data_classes import MigrationMappingList
-
-
-class MigrateTimeseriesCommand(BaseMigrateCommand):
-    cdf_cdm = "cdf_cdm"
-    view_id = ViewId(cdf_cdm, "CogniteTimeSeries", "v1")
-    cdf_cdm_units = "cdf_cdm_units"
-    chunk_size = 1000
-
-    @property
-    def schema_spaces(self) -> list[str]:
-        return [self.cdf_cdm, self.cdf_cdm_units]
-
-    def source_acl(self, data_set_ids: list[int]) -> Capability:
-        return TimeSeriesAcl(
-            actions=[TimeSeriesAcl.Action.Read, TimeSeriesAcl.Action.Write],
-            scope=DataSetScope(data_set_ids),
-        )
-
-    def migrate_timeseries(
-        self,
-        client: ToolkitClient,
-        mapping_file: Path,
-        dry_run: bool = False,
-        verbose: bool = False,
-        auto_yes: bool = False,
-    ) -> None:
-        """Migrate resources from Asset-Centric to data modeling in CDF."""
-        mappings = MigrationMappingList.read_mapping_file(mapping_file)
-        self.validate_access(client, list(mappings.spaces()), list(mappings.get_data_set_ids()))
-        self._validate_timeseries_existence(client, mappings)
-        self.validate_available_capacity(client, len(mappings))
-
-        if dry_run:
-            self.console(f"Dry run mode. Would have migrated {len(mappings):,} TimeSeries to CogniteTimeSeries.")
-            return
-        if not auto_yes and self._confirm(mappings) is False:
-            return
-        self._migrate(client, mappings, verbose)
-
-    def _validate_timeseries_existence(self, client: ToolkitClient, mappings: MigrationMappingList) -> None:
-        total_validated = 0
-        chunk: MigrationMappingList
-        for chunk in track(  # type: ignore[assignment]
-            chunker_sequence(mappings, size=self.chunk_size),
-            description="Validating...",
-            total=len(mappings) // self.chunk_size + 1,
-        ):
-            try:
-                timeseries = client.time_series.retrieve_multiple(
-                    ids=chunk.get_ids(),
-                    external_ids=chunk.get_external_ids(),
-                    ignore_unknown_ids=True,
-                )
-            except CogniteAPIError as e:
-                raise AuthenticationError(
-                    f"Failed to retrieve TimeSeries. This is likely due to lack of permissions: {e!s}"
-                ) from e
-
-            missing_count = len(timeseries) - len(mappings)
-            if missing_count > 0:
-                raise ToolkitValueError(f"Missing {missing_count} TimeSeries does not exist in CDF.")
-
-            existing_result = client.data_modeling.instances.retrieve(chunk.as_node_ids())
-            if len(existing_result.nodes) != 0:
-                raise ToolkitValueError(
-                    "Some of the TimeSeries you are trying to migrate already exist in Data Modeling. "
-                    f"Please remove the following TimeSeries from the mapping file {humanize_collection(existing_result.nodes.as_ids())}"
-                )
-            total_validated += len(timeseries)
-        print(
-            f"Validated {total_validated:,} TimeSeries for migration. "
-            f"{len(mappings):,} mappings provided in the mapping file."
-        )
-
-    @staticmethod
-    def _confirm(mappings: MigrationMappingList) -> bool:
-        print(
-            Panel(
-                f"[red]WARNING:[/red] This operation [bold]cannot be undone[/bold]! "
-                f"{len(mappings):,} TimeSeries will linked to the new CogniteTimeSeries. "
-                "This linking cannot be undone",
-                style="bold",
-                title="Migrate asset-centric TimeSeries to CogniteTimeSeries",
-                title_align="left",
-                border_style="red",
-                expand=False,
-            )
-        )
-
-        if not questionary.confirm("Are you really sure you want to continue?", default=False).ask():
-            print("Migration cancelled by user.")
-            return False
-        return True
-
-    def _migrate(self, client: ToolkitClient, mappings: MigrationMappingList, verbose: bool) -> None:
-        print("Migrating TimeSeries to CogniteTimeSeries...")
-        total_migrated = 0
-        for chunk in track(
-            chunker_sequence(mappings, size=self.chunk_size),
-            description="Migrating TimeSeries to CogniteTimeSeries...",
-            total=len(mappings) // self.chunk_size + 1,
-        ):  # type: ignore[assignment]
-            if verbose:
-                print(f"Migrating {len(chunk):,} TimeSeries...")
-
-            # Set pending IDs for the chunk of mappings
-            try:
-                pending_timeseries = client.time_series.set_pending_ids(chunk.as_pending_ids())
-            except CogniteAPIError as e:
-                raise ToolkitValueError(f"Failed to set pending IDs for TimeSeries: {e!s}") from e
-
-            # The ExtendedTimeSeriesList is iterating ExtendedTimeSeries objects.
-            converted_timeseries = [self.as_cognite_timeseries(ts) for ts in pending_timeseries]  # type: ignore[arg-type]
-            try:
-                created = client.data_modeling.instances.apply_fast(converted_timeseries)
-            except CogniteAPIError as e:
-                raise ToolkitValueError(f"Failed to apply TimeSeries: {e!s}") from e
-            if verbose:
-                print(f"Created {len(created):,} CogniteTimeSeries.")
-            total_migrated += len(created)
-        print(f"Successfully migrated {total_migrated:,} TimeSeries to CogniteTimeSeries.")
-
-    @classmethod
-    def as_cognite_timeseries(cls, ts: ExtendedTimeSeries) -> CogniteTimeSeriesApply:
-        if ts.pending_instance_id is None:
-            raise ToolkitValueError("ExtendedTimeSeries must have a pending_instance_id set before migration.")
-        if ts.is_step is None:
-            raise ToolkitValueError("ExtendedTimeSeries must have is_step set before migration.")
-
-        return CogniteTimeSeriesApply(
-            space=ts.pending_instance_id.space,
-            external_id=ts.pending_instance_id.external_id,
-            name=ts.name,
-            description=ts.description,
-            is_step=ts.is_step,
-            time_series_type="string" if ts.is_string else "numeric",
-            source_unit=ts.unit,
-            unit=DirectRelationReference(cls.cdf_cdm_units, ts.unit_external_id) if ts.unit_external_id else None,
-        )