cognite-toolkit 0.6.97__py3-none-any.whl → 0.7.39__py3-none-any.whl

cognite_toolkit/_cdf_tk/apps/_migrate_app.py

@@ -4,34 +4,51 @@ from typing import Annotated, Any
 
 import questionary
 import typer
+from cognite.client.data_classes import Annotation
 from cognite.client.data_classes.data_modeling import ContainerId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.commands import (
-    MigrationCanvasCommand,
-    MigrationPrepareCommand,
-)
+from cognite_toolkit._cdf_tk.commands import MigrationPrepareCommand
 from cognite_toolkit._cdf_tk.commands._migrate import MigrationCommand
 from cognite_toolkit._cdf_tk.commands._migrate.creators import (
     InfieldV2ConfigCreator,
     InstanceSpaceCreator,
     SourceSystemCreator,
 )
-from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper
+from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import (
+    AssetCentricMapper,
+    CanvasMapper,
+    ChartMapper,
+    ThreeDAssetMapper,
+    ThreeDMapper,
+)
 from cognite_toolkit._cdf_tk.commands._migrate.migration_io import (
+    AnnotationMigrationIO,
     AssetCentricMigrationIO,
+    ThreeDAssetMappingMigrationIO,
+    ThreeDMigrationIO,
 )
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import (
     AssetCentricMigrationSelector,
     MigrateDataSetSelector,
     MigrationCSVFileSelector,
 )
+from cognite_toolkit._cdf_tk.storageio import CanvasIO, ChartIO
+from cognite_toolkit._cdf_tk.storageio.selectors import (
+    CanvasExternalIdSelector,
+    ChartExternalIdSelector,
+    ThreeDModelIdSelector,
+)
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._cdf_tk.utils.cli_args import parse_view_str
 from cognite_toolkit._cdf_tk.utils.interactive_select import (
     AssetInteractiveSelect,
     DataModelingSelect,
+    FileMetadataInteractiveSelect,
+    InteractiveCanvasSelect,
+    InteractiveChartSelect,
     ResourceViewMappingInteractiveSelect,
+    ThreeDInteractiveSelect,
 )
 from cognite_toolkit._cdf_tk.utils.useful_types import AssetCentricKind
 
@@ -49,7 +66,11 @@ class MigrateApp(typer.Typer):
         self.command("events")(self.events)
         self.command("timeseries")(self.timeseries)
         self.command("files")(self.files)
+        self.command("annotations")(self.annotations)
         self.command("canvas")(self.canvas)
+        self.command("charts")(self.charts)
+        self.command("3d")(self.three_d)
+        self.command("3d-mappings")(self.three_d_asset_mapping)
         # Uncomment when infield v2 config migration is ready
         # self.command("infield-configs")(self.infield_configs)
 
@@ -694,6 +715,149 @@ class MigrateApp(typer.Typer):
             )
         )
 
+    @classmethod
+    def annotations(
+        cls,
+        ctx: typer.Context,
+        mapping_file: Annotated[
+            Path | None,
+            typer.Option(
+                "--mapping-file",
+                "-m",
+                help="Path to the mapping file that contains the mapping from Annotations to CogniteDiagramAnnotation. "
+                "This file is expected to have the following columns: [id, space, externalId, ingestionView].",
+            ),
+        ] = None,
+        data_set_id: Annotated[
+            str | None,
+            typer.Option(
+                "--data-set-id",
+                "-s",
+                help="The data set ID to select for the annotations to migrate. If not provided and the mapping file is not provided, "
+                "an interactive selection will be performed to select the data set to migrate annotations from.",
+            ),
+        ] = None,
+        instance_space: Annotated[
+            str | None,
+            typer.Option(
+                "--instance-space",
+                "-i",
+                help="The instance space to use for the migrated annotations. Required when using --data-set-id.",
+            ),
+        ] = None,
+        asset_annotation_mapping: Annotated[
+            str | None,
+            typer.Option(
+                "--asset-annotation-mapping",
+                "-a",
+                help="The ingestion mapping to use for asset-linked annotations. If not provided, "
+                "the default mapping (cdf_asset_annotations_mapping) will be used.",
+            ),
+        ] = None,
+        file_annotation_mapping: Annotated[
+            str | None,
+            typer.Option(
+                "--file-annotation-mapping",
+                "-f",
+                help="The ingestion mapping to use for file-linked annotations. If not provided, "
+                "the default mapping (cdf_file_annotations_mapping) will be used.",
+            ),
+        ] = None,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where logs will be stored. If the directory does not exist, it will be created.",
+            ),
+        ] = Path(f"migration_logs_{TODAY!s}"),
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the migration will not be executed, but only a report of what would be done is printed.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Migrate Annotations to CogniteDiagramAnnotation edges in data modeling.
+
+        Annotations are diagram annotations that link assets or files to other resources. This command
+        migrates them to edges in the data modeling space, preserving the relationships and metadata.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+
+        if data_set_id is not None and mapping_file is not None:
+            raise typer.BadParameter("Cannot specify both data_set_id and mapping_file")
+        elif mapping_file is not None:
+            selected: AssetCentricMigrationSelector = MigrationCSVFileSelector(
+                datafile=mapping_file, kind="Annotations"
+            )
+            annotation_io = AnnotationMigrationIO(client)
+        elif data_set_id is not None:
+            if instance_space is None:
+                raise typer.BadParameter("--instance-space is required when using --data-set-id")
+            selected = MigrateDataSetSelector(data_set_external_id=data_set_id, kind="Annotations")
+            annotation_io = AnnotationMigrationIO(
+                client,
+                instance_space=instance_space,
+                default_asset_annotation_mapping=asset_annotation_mapping,
+                default_file_annotation_mapping=file_annotation_mapping,
+            )
+        else:
+            # Interactive selection
+            selector = FileMetadataInteractiveSelect(client, "migrate")
+            selected_data_set_id = selector.select_data_set(allow_empty=False)
+            dm_selector = DataModelingSelect(client, "migrate")
+            selected_instance_space = dm_selector.select_instance_space(
+                multiselect=False,
+                message="In which instance space do you want to create the annotations?",
+                include_empty=True,
+            )
+            if selected_instance_space is None:
+                raise typer.Abort()
+            asset_annotations_selector = ResourceViewMappingInteractiveSelect(client, "migrate asset annotations")
+            asset_annotation_mapping = asset_annotations_selector.select_resource_view_mapping(
+                resource_type="assetAnnotation",
+            ).external_id
+            file_annotations_selector = ResourceViewMappingInteractiveSelect(client, "migrate file annotations")
+            file_annotation_mapping = file_annotations_selector.select_resource_view_mapping(
+                resource_type="fileAnnotation",
+            ).external_id
+
+            selected = MigrateDataSetSelector(data_set_external_id=selected_data_set_id, kind="Annotations")
+            annotation_io = AnnotationMigrationIO(
+                client,
+                instance_space=selected_instance_space,
+                default_asset_annotation_mapping=asset_annotation_mapping,
+                default_file_annotation_mapping=file_annotation_mapping,
+            )
+
+            dry_run = questionary.confirm("Do you want to perform a dry run?", default=dry_run).ask()
+            verbose = questionary.confirm("Do you want verbose output?", default=verbose).ask()
+            if any(res is None for res in [dry_run, verbose]):
+                raise typer.Abort()
+
+        cmd = MigrationCommand()
+        cmd.run(
+            lambda: cmd.migrate(
+                selected=selected,
+                data=annotation_io,
+                mapper=AssetCentricMapper[Annotation](client),
+                log_dir=log_dir,
+                dry_run=dry_run,
+                verbose=verbose,
+            )
+        )
+
     @staticmethod
     def canvas(
         ctx: typer.Context,
@@ -704,6 +868,23 @@ class MigrateApp(typer.Typer):
                 "performed to select the Canvas to migrate."
             ),
         ] = None,
+        skip_on_missing_ref: Annotated[
+            bool,
+            typer.Option(
+                "--skip-on-missing-ref",
+                "-s",
+                help="If set, the migration will skip Canvases that reference resources that have not been migrated to data modeling. "
+                "If not set, the migration will continue but the result will exclude the missing references.",
+            ),
+        ] = False,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where migration logs will be stored.",
+            ),
+        ] = Path(f"migration_logs_{TODAY}"),
         dry_run: Annotated[
             bool,
             typer.Option(
@@ -729,12 +910,246 @@ class MigrateApp(typer.Typer):
         is populated with the mapping from Asset-Centric resources to the new data modeling resources.
         """
         client = EnvironmentVariables.create_from_environment().get_client()
+        if external_id is None:
+            interactive = InteractiveCanvasSelect(client)
+            external_id = interactive.select_external_ids()
+            log_dir = questionary.path("Specify log directory for migration logs:", default=str(log_dir)).ask()
+            dry_run = questionary.confirm("Do you want to perform a dry run?", default=dry_run).ask()
+            verbose = questionary.confirm("Do you want verbose output?", default=verbose).ask()
+            if any(res is None for res in [log_dir, dry_run, verbose]):
+                raise typer.Abort()
+            log_dir = Path(log_dir)
 
-        cmd = MigrationCanvasCommand()
+        cmd = MigrationCommand()
+        selector = CanvasExternalIdSelector(external_ids=tuple(external_id))
         cmd.run(
-            lambda: cmd.migrate_canvas(
-                client,
-                external_ids=external_id,
+            lambda: cmd.migrate(
+                selected=selector,
+                data=CanvasIO(client, exclude_existing_version=True),
+                mapper=CanvasMapper(client, dry_run=dry_run, skip_on_missing_ref=skip_on_missing_ref),
+                log_dir=log_dir,
+                dry_run=dry_run,
+                verbose=verbose,
+            )
+        )
+
+    @staticmethod
+    def charts(
+        ctx: typer.Context,
+        external_id: Annotated[
+            list[str] | None,
+            typer.Argument(
+                help="The external ID of the Chart to migrate. If not provided, an interactive selection will be "
+                "performed to select the Charts to migrate."
+            ),
+        ] = None,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where migration logs will be stored.",
+            ),
+        ] = Path(f"migration_logs_{TODAY}"),
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the migration will not be executed, but only a report of "
+                "what would be done is printed. This is useful for checking that all time series referenced by the Charts "
+                "have been migrated to the new data modeling resources in CDF.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Migrate Charts from time series references to data modeling in CDF.
+
+        This command expects that the CogniteMigration data model is already deployed, and that the Mapping view
+        is populated with the mapping from time series to the new data modeling resources.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+
+        selected_external_ids: list[str]
+        if external_id:
+            selected_external_ids = external_id
+        else:
+            selected_external_ids = InteractiveChartSelect(client).select_external_ids()
+
+        cmd = MigrationCommand()
+        cmd.run(
+            lambda: cmd.migrate(
+                selected=ChartExternalIdSelector(external_ids=tuple(selected_external_ids)),
+                data=ChartIO(client),
+                mapper=ChartMapper(client),
+                log_dir=log_dir,
+                dry_run=dry_run,
+                verbose=verbose,
+            )
+        )
+
+    @staticmethod
+    def three_d(
+        ctx: typer.Context,
+        id: Annotated[
+            list[int] | None,
+            typer.Argument(
+                help="The ID of the 3D Model to migrate. If not provided, an interactive selection will be "
+                "performed to select the 3D Models to migrate."
+            ),
+        ] = None,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where migration logs will be stored.",
+            ),
+        ] = Path(f"migration_logs_{TODAY}"),
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the migration will not be executed, but only a report of "
+                "what would be done is printed. This is useful for checking that all resources referenced by the 3D Models "
+                "have been migrated to the new data modeling resources in CDF.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Migrate 3D Models from Asset-Centric to data modeling in CDF.
+
+        This command expects that the CogniteMigration data model is already deployed, and that the Mapping view
+        is populated with the mapping from Asset-Centric resources to the new data modeling resources.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+        selected_ids: list[int]
+        if id:
+            selected_ids = id
+        else:
+            selected_models = ThreeDInteractiveSelect(client, "migrate").select_three_d_models("classic")
+            selected_ids = [model.id for model in selected_models]
+
+        cmd = MigrationCommand()
+        cmd.run(
+            lambda: cmd.migrate(
+                selected=ThreeDModelIdSelector(ids=tuple(selected_ids)),
+                data=ThreeDMigrationIO(client),
+                mapper=ThreeDMapper(client),
+                log_dir=log_dir,
+                dry_run=dry_run,
+                verbose=verbose,
+            )
+        )
+
+    @staticmethod
+    def three_d_asset_mapping(
+        ctx: typer.Context,
+        model_id: Annotated[
+            list[int] | None,
+            typer.Argument(
+                help="The IDs of the 3D model to migrate asset mappings for. If not provided, an interactive selection will be "
+                "performed to select the."
+            ),
+        ] = None,
+        object_3D_space: Annotated[
+            str | None,
+            typer.Option(
+                "--object-3d-space",
+                "-o",
+                help="The instance space to ceate the 3D object nodes in.",
+            ),
+        ] = None,
+        cad_node_space: Annotated[
+            str | None,
+            typer.Option(
+                "--cad-node-space",
+                "-c",
+                help="The instance space to create the CAD node nodes in.",
+            ),
+        ] = None,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where migration logs will be stored.",
+            ),
+        ] = Path(f"migration_logs_{TODAY}"),
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the migration will not be executed, but only a report of what would be done is printed.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Migrate 3D Model Asset Mappings from Asset-Centric to data modeling in CDF.
+
+        This command expects that the selected 3D model has already been migrated to data modeling.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+        selected_ids: list[int]
+        if model_id is not None:
+            selected_ids = model_id
+        else:
+            # Interactive selection
+            selected_models = ThreeDInteractiveSelect(client, "migrate").select_three_d_models("dm")
+            selected_ids = [model.id for model in selected_models]
+            space_selector = DataModelingSelect(client, "migrate")
+            object_3D_space = space_selector.select_instance_space(
+                multiselect=False,
+                message="In which instance space do you want to create the 3D Object nodes?",
+                include_empty=False,
+            )
+            cad_node_space = space_selector.select_instance_space(
+                multiselect=False,
+                message="In which instance space do you want to create the CAD Node nodes?",
+                include_empty=False,
+            )
+            dry_run = questionary.confirm("Do you want to perform a dry run?", default=dry_run).ask()
+            verbose = questionary.confirm("Do you want verbose output?", default=verbose).ask()
+            if any(res is None for res in [dry_run, verbose]):
+                raise typer.Abort()
+
+        if object_3D_space is None or cad_node_space is None:
+            raise typer.BadParameter(
+                "--object-3d-space and --cad-node-space are required when specifying IDs directly."
+            )
+
+        cmd = MigrationCommand()
+        cmd.run(
+            lambda: cmd.migrate(
+                selected=ThreeDModelIdSelector(ids=tuple(selected_ids)),
+                data=ThreeDAssetMappingMigrationIO(
+                    client, object_3D_space=object_3D_space, cad_node_space=cad_node_space
+                ),
+                mapper=ThreeDAssetMapper(client),
+                log_dir=log_dir,
                 dry_run=dry_run,
                 verbose=verbose,
             )

cognite_toolkit/_cdf_tk/apps/_modules_app.py

@@ -6,7 +6,6 @@ from rich import print
 
 from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
 from cognite_toolkit._cdf_tk.commands import ModulesCommand, PullCommand
-from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._version import __version__
 
@@ -58,7 +57,6 @@ class ModulesApp(typer.Typer):
                 "--library-url",
                 "-u",
                 help="URL of the library to add to the project.",
-                hidden=not Flags.EXTERNAL_LIBRARIES.is_enabled(),
             ),
         ] = None,
         library_checksum: Annotated[
@@ -67,7 +65,6 @@ class ModulesApp(typer.Typer):
                 "--library-checksum",
                 "-c",
                 help="Checksum of the library to add to the project.",
-                hidden=not Flags.EXTERNAL_LIBRARIES.is_enabled(),
             ),
         ] = None,
         verbose: Annotated[

cognite_toolkit/_cdf_tk/apps/_purge.py

@@ -8,7 +8,6 @@ from rich import print
 
 from cognite_toolkit._cdf_tk.commands import PurgeCommand
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
-from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.storageio.selectors import (
     InstanceFileSelector,
     InstanceSelector,
@@ -31,8 +30,7 @@ class PurgeApp(typer.Typer):
         self.callback(invoke_without_command=True)(self.main)
         self.command("dataset")(self.purge_dataset)
         self.command("space")(self.purge_space)
-        if Flags.PURGE_INSTANCES.is_enabled() or Flags.v07.is_enabled():
-            self.command("instances")(self.purge_instances)
+        self.command("instances")(self.purge_instances)
 
     def main(self, ctx: typer.Context) -> None:
         """Commands deleting data from Cognite Data Fusion."""
@@ -48,21 +46,11 @@ class PurgeApp(typer.Typer):
                 help="External id of the dataset to purge. If not provided, interactive mode will be used.",
             ),
         ] = None,
-        include_dataset: Annotated[
-            bool,
-            typer.Option(
-                "--include-dataset",
-                "-i",
-                help="Whether to archive the dataset itself after purging its contents.",
-                hidden=Flags.v07.is_enabled(),
-            ),
-        ] = False,
         archive_dataset: Annotated[
             bool,
             typer.Option(
                 "--archive-dataset",
                 help="Whether to archive the dataset itself after purging its contents.",
-                hidden=not Flags.v07.is_enabled(),
             ),
         ] = False,
         skip_data: Annotated[
@@ -73,7 +61,6 @@ class PurgeApp(typer.Typer):
                 help="Skip deleting the data in the dataset, only delete configurations. The resources that are "
                 "considered data are: time series, event, files, assets, sequences, relationships, "
                 "labels, and 3D Models",
-                hidden=not Flags.v07.is_enabled(),
             ),
         ] = False,
         include_configurations: Annotated[
@@ -82,7 +69,6 @@ class PurgeApp(typer.Typer):
                 "--include-configurations",
                 "-c",
                 help="Include configurations, workflows, extraction pipelines and transformations in the purge.",
-                hidden=not Flags.v07.is_enabled(),
             ),
         ] = False,
         asset_recursive: Annotated[
@@ -92,7 +78,6 @@ class PurgeApp(typer.Typer):
                 "-a",
                 help="When deleting assets, delete all child assets recursively. CAVEAT: This can lead to assets"
                 " not in the selected dataset being deleted if they are children of assets in the dataset.",
-                hidden=not Flags.v07.is_enabled(),
             ),
         ] = False,
         dry_run: Annotated[
@@ -128,39 +113,28 @@ class PurgeApp(typer.Typer):
             # Is Interactive
             interactive = AssetInteractiveSelect(client, operation="purge")
             external_id = interactive.select_data_set(allow_empty=False)
-            if Flags.v07.is_enabled():
-                skip_data = not questionary.confirm(
-                    "Delete data in the dataset (time series, events, files, assets, sequences, relationships, labels, 3D models)?",
-                    default=True,
-                ).ask()
-                include_configurations = questionary.confirm(
-                    "Delete configurations (workflows, extraction pipelines and transformations) in the dataset?",
-                    default=False,
-                ).ask()
-                asset_recursive = questionary.confirm(
-                    "When deleting assets, delete all child assets recursively? (WARNING: This can lead "
-                    "to assets not in the selected dataset being deleted if they are children of assets in the dataset.)",
-                    default=False,
-                ).ask()
+            skip_data = not questionary.confirm(
+                "Delete data in the dataset (time series, events, files, assets, sequences, relationships, labels, 3D models)?",
+                default=True,
+            ).ask()
+            include_configurations = questionary.confirm(
+                "Delete configurations (workflows, extraction pipelines and transformations) in the dataset?",
+                default=False,
+            ).ask()
+            asset_recursive = questionary.confirm(
+                "When deleting assets, delete all child assets recursively? (WARNING: This can lead "
+                "to assets not in the selected dataset being deleted if they are children of assets in the dataset.)",
+                default=False,
+            ).ask()
             archive_dataset = questionary.confirm("Archive the dataset itself after purging?", default=False).ask()
             dry_run = questionary.confirm("Dry run?", default=True).ask()
             verbose = questionary.confirm("Verbose?", default=True).ask()
 
-            user_options = [archive_dataset, dry_run, verbose]
-            if Flags.v07.is_enabled():
-                user_options.extend([skip_data, include_configurations, asset_recursive])
+            user_options = [archive_dataset, dry_run, verbose, skip_data, include_configurations, asset_recursive]
 
             if any(selected is None for selected in user_options):
                 raise typer.Abort("Aborted by user.")
 
-        else:
-            archive_dataset = archive_dataset if Flags.v07.is_enabled() else include_dataset
-
-        if not Flags.v07.is_enabled():
-            skip_data = False
-            include_configurations = True
-            asset_recursive = False
-
         cmd.run(
             lambda: cmd.dataset(
                 client,
@@ -197,7 +171,6 @@ class PurgeApp(typer.Typer):
             typer.Option(
                 "--delete-datapoints",
                 help="Delete datapoints linked to CogniteTimeSeries nodes in the space.",
-                hidden=not Flags.v07.is_enabled(),
             ),
         ] = False,
         delete_file_content: Annotated[
@@ -205,7 +178,6 @@ class PurgeApp(typer.Typer):
             typer.Option(
                 "--delete-file-content",
                 help="Delete file content linked to CogniteFile nodes in the space.",
-                hidden=not Flags.v07.is_enabled(),
             ),
         ] = False,
         dry_run: Annotated[