cognite-toolkit 0.6.97__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_cdf_tk/apps/_dump_app.py

@@ -7,13 +7,7 @@ from cognite.client.data_classes.data_modeling import DataModelId, ViewId
 from rich import print
 
 from cognite_toolkit._cdf_tk.client.data_classes.search_config import ViewId as SearchConfigViewId
-from cognite_toolkit._cdf_tk.commands import DumpDataCommand, DumpResourceCommand
-from cognite_toolkit._cdf_tk.commands.dump_data import (
-    AssetFinder,
-    EventFinder,
-    FileMetadataFinder,
-    TimeSeriesFinder,
-)
+from cognite_toolkit._cdf_tk.commands import DumpResourceCommand
 from cognite_toolkit._cdf_tk.commands.dump_resource import (
     AgentFinder,
     DataModelFinder,
@@ -32,45 +26,32 @@ from cognite_toolkit._cdf_tk.commands.dump_resource import (
 from cognite_toolkit._cdf_tk.exceptions import ToolkitRequiredValueError
 from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
-from cognite_toolkit._cdf_tk.utils.interactive_select import (
-    AssetInteractiveSelect,
-    EventInteractiveSelect,
-    FileMetadataInteractiveSelect,
-    TimeSeriesInteractiveSelect,
-)
 
 
 class DumpApp(typer.Typer):
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.callback(invoke_without_command=True)(self.dump_main)
-        if Flags.DUMP_DATA.is_enabled():
-            self.add_typer(DumpDataApp(*args, **kwargs), name="data")
-            self.add_typer(DumpConfigApp(*args, **kwargs), name="config")
-        else:
-            self.command("datamodel")(DumpConfigApp.dump_datamodel_cmd)
+        self.command("datamodel")(DumpConfigApp.dump_datamodel_cmd)
 
-            self.command("asset")(DumpDataApp.dump_asset_cmd)
-            self.command("timeseries")(DumpDataApp.dump_timeseries_cmd)
+        self.command("asset")(DumpDataApp.dump_asset_cmd)
+        self.command("timeseries")(DumpDataApp.dump_timeseries_cmd)
 
-            self.command("workflow")(DumpConfigApp.dump_workflow)
-            self.command("transformation")(DumpConfigApp.dump_transformation)
-            self.command("group")(DumpConfigApp.dump_group)
-            self.command("node")(DumpConfigApp.dump_node)
-            self.command("spaces")(DumpConfigApp.dump_spaces)
+        self.command("workflow")(DumpConfigApp.dump_workflow)
+        self.command("transformation")(DumpConfigApp.dump_transformation)
+        self.command("group")(DumpConfigApp.dump_group)
+        self.command("node")(DumpConfigApp.dump_node)
+        self.command("spaces")(DumpConfigApp.dump_spaces)
 
-            if Flags.DUMP_EXTENDED.is_enabled():
-                self.command("location-filter")(DumpConfigApp.dump_location_filters)
-                self.command("extraction-pipeline")(DumpConfigApp.dump_extraction_pipeline)
-                self.command("functions")(DumpConfigApp.dump_functions)
-                self.command("datasets")(DumpConfigApp.dump_datasets)
-                self.command("streamlit")(DumpConfigApp.dump_streamlit)
+        self.command("location-filter")(DumpConfigApp.dump_location_filters)
+        self.command("extraction-pipeline")(DumpConfigApp.dump_extraction_pipeline)
+        self.command("functions")(DumpConfigApp.dump_functions)
+        self.command("datasets")(DumpConfigApp.dump_datasets)
+        self.command("streamlit")(DumpConfigApp.dump_streamlit)
 
-                if Flags.AGENTS.is_enabled():
-                    self.command("agents")(DumpConfigApp.dump_agents)
+        self.command("agents")(DumpConfigApp.dump_agents)
 
-                if Flags.SEARCH_CONFIG.is_enabled():
-                    self.command("search-config")(DumpConfigApp.dump_search_config)
+        self.command("search-config")(DumpConfigApp.dump_search_config)
 
     @staticmethod
     def dump_main(ctx: typer.Context) -> None:
@@ -91,18 +72,13 @@ class DumpConfigApp(typer.Typer):
         self.command("group")(self.dump_group)
         self.command("node")(self.dump_node)
         self.command("spaces")(self.dump_spaces)
-        if Flags.DUMP_EXTENDED.is_enabled():
-            self.command("location-filters")(self.dump_location_filters)
-            self.command("extraction-pipeline")(self.dump_extraction_pipeline)
-            self.command("datasets")(DumpConfigApp.dump_datasets)
-            self.command("functions")(self.dump_functions)
-            self.command("streamlit")(DumpConfigApp.dump_streamlit)
-
-            if Flags.AGENTS.is_enabled():
-                self.command("agents")(self.dump_agents)
-
-            if Flags.SEARCH_CONFIG.is_enabled():
-                self.command("search-config")(self.dump_search_config)
+        self.command("location-filters")(self.dump_location_filters)
+        self.command("extraction-pipeline")(self.dump_extraction_pipeline)
+        self.command("datasets")(DumpConfigApp.dump_datasets)
+        self.command("functions")(self.dump_functions)
+        self.command("streamlit")(DumpConfigApp.dump_streamlit)
+        self.command("agents")(self.dump_agents)
+        self.command("search-config")(self.dump_search_config)
 
     @staticmethod
     def dump_config_main(ctx: typer.Context) -> None:
@@ -862,21 +838,11 @@ class DumpDataApp(typer.Typer):
         ] = False,
     ) -> None:
         """This command will dump the selected assets in the selected format to the folder specified, defaults to /tmp."""
-        cmd = DumpDataCommand()
-        client = EnvironmentVariables.create_from_environment().get_client()
-        if hierarchy is None and data_set is None:
-            hierarchy, data_set = AssetInteractiveSelect(client, "dump").select_hierarchies_and_data_sets()
-
-        cmd.run(
-            lambda: cmd.dump_table(
-                AssetFinder(client, hierarchy or [], data_set or []),
-                output_dir,
-                clean,
-                limit,
-                format_,
-                verbose,
+        if Flags.v08:
+            raise ValueError(
+                "The `cdf dump data asset` command has been removed. Please use `cdf data download assets` instead."
             )
-        )
+        print("[bold yellow]Warning:[/] This command has been removed. Please use `cdf data download assets` instead.")
 
     @staticmethod
     def dump_files_cmd(
@@ -940,21 +906,12 @@ class DumpDataApp(typer.Typer):
         ] = False,
     ) -> None:
         """This command will dump the selected events to the selected format in the folder specified, defaults to /tmp."""
-        cmd = DumpDataCommand()
-        cmd.validate_directory(output_dir, clean)
-        client = EnvironmentVariables.create_from_environment().get_client()
-        if hierarchy is None and data_set is None:
-            hierarchy, data_set = FileMetadataInteractiveSelect(client, "dump").select_hierarchies_and_data_sets()
-        cmd.run(
-            lambda: cmd.dump_table(
-                FileMetadataFinder(client, hierarchy or [], data_set or []),
-                output_dir,
-                clean,
-                limit,
-                format_,
-                verbose,
+        if Flags.v08:
+            raise ValueError(
+                "The `cdf dump data files-metadata` command has been removed. Please use `cdf data download files` instead."
             )
-        )
+        print("[bold yellow]Warning:[/] This command has been removed. Please use `cdf data download files` instead.")
+        return None
 
     @staticmethod
     def dump_timeseries_cmd(
@@ -1018,20 +975,14 @@ class DumpDataApp(typer.Typer):
         ] = False,
     ) -> None:
         """This command will dump the selected timeseries to the selected format in the folder specified, defaults to /tmp."""
-        cmd = DumpDataCommand()
-        client = EnvironmentVariables.create_from_environment().get_client()
-        if hierarchy is None and data_set is None:
-            hierarchy, data_set = TimeSeriesInteractiveSelect(client, "dump").select_hierarchies_and_data_sets()
-        cmd.run(
-            lambda: cmd.dump_table(
-                TimeSeriesFinder(client, hierarchy or [], data_set or []),
-                output_dir,
-                clean,
-                limit,
-                format_,
-                verbose,
+        if Flags.v08:
+            raise ValueError(
+                "The `cdf dump data timeseries` command has been removed. Please use `cdf data download timeseries` instead."
             )
+        print(
+            "[bold yellow]Warning:[/] This command has been removed. Please use `cdf data download timeseries` instead."
         )
+        return None
 
     @staticmethod
     def dump_event_cmd(
@@ -1095,18 +1046,9 @@ class DumpDataApp(typer.Typer):
         ] = False,
     ) -> None:
         """This command will dump the selected events to the selected format in the folder specified, defaults to /tmp."""
-        cmd = DumpDataCommand()
-        cmd.validate_directory(output_dir, clean)
-        client = EnvironmentVariables.create_from_environment().get_client()
-        if hierarchy is None and data_set is None:
-            hierarchy, data_set = EventInteractiveSelect(client, "dump").select_hierarchies_and_data_sets()
-        cmd.run(
-            lambda: cmd.dump_table(
-                EventFinder(client, hierarchy or [], data_set or []),
-                output_dir,
-                clean,
-                limit,
-                format_,
-                verbose,
+        if Flags.v08:
+            raise ValueError(
+                "The `cdf dump data event` command has been removed. Please use `cdf data download events` instead."
             )
-        )
+        print("[bold yellow]Warning:[/] This command has been removed. Please use `cdf data download events` instead.")
+        return None

cognite_toolkit/_cdf_tk/apps/_landing_app.py

@@ -1,3 +1,5 @@
+from typing import Annotated
+
 import typer
 
 from cognite_toolkit._cdf_tk.commands import InitCommand
@@ -6,9 +8,21 @@ from cognite_toolkit._cdf_tk.commands import InitCommand
 class LandingApp(typer.Typer):
     def __init__(self, *args, **kwargs) -> None:  # type: ignore
         super().__init__(*args, **kwargs)
-        self.command()(self.main_init)
 
-    def main_init(self) -> None:
-        """Guidance on how to get started"""
+    def main_init(
+        self,
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-r",
+                help="Whether to do a dry-run, do dry-run if present.",
+            ),
+        ] = False,
+    ) -> None:
+        """Getting started checklist"""
         cmd = InitCommand()
-        cmd.run(cmd.execute)
+        # Tracking  command with the usual lambda run construct
+        # is intentionally left out because we don't want to expose the user to the warning
+        # before they've had the chance to opt in (which is something they'll do later using this command).
+        cmd.execute(dry_run=dry_run)

cognite_toolkit/_cdf_tk/apps/_migrate_app.py

@@ -4,21 +4,20 @@ from typing import Annotated, Any
 
 import questionary
 import typer
+from cognite.client.data_classes import Annotation
 from cognite.client.data_classes.data_modeling import ContainerId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.commands import (
-    MigrationCanvasCommand,
-    MigrationPrepareCommand,
-)
+from cognite_toolkit._cdf_tk.commands import MigrationPrepareCommand
 from cognite_toolkit._cdf_tk.commands._migrate import MigrationCommand
 from cognite_toolkit._cdf_tk.commands._migrate.creators import (
     InfieldV2ConfigCreator,
     InstanceSpaceCreator,
     SourceSystemCreator,
 )
-from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper
+from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper, CanvasMapper, ChartMapper
 from cognite_toolkit._cdf_tk.commands._migrate.migration_io import (
+    AnnotationMigrationIO,
     AssetCentricMigrationIO,
 )
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import (
@@ -26,11 +25,16 @@ from cognite_toolkit._cdf_tk.commands._migrate.selectors import (
     MigrateDataSetSelector,
     MigrationCSVFileSelector,
 )
+from cognite_toolkit._cdf_tk.storageio import CanvasIO, ChartIO
+from cognite_toolkit._cdf_tk.storageio.selectors import CanvasExternalIdSelector, ChartExternalIdSelector
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._cdf_tk.utils.cli_args import parse_view_str
 from cognite_toolkit._cdf_tk.utils.interactive_select import (
     AssetInteractiveSelect,
     DataModelingSelect,
+    FileMetadataInteractiveSelect,
+    InteractiveCanvasSelect,
+    InteractiveChartSelect,
     ResourceViewMappingInteractiveSelect,
 )
 from cognite_toolkit._cdf_tk.utils.useful_types import AssetCentricKind
@@ -49,7 +53,9 @@ class MigrateApp(typer.Typer):
         self.command("events")(self.events)
         self.command("timeseries")(self.timeseries)
         self.command("files")(self.files)
+        self.command("annotations")(self.annotations)
         self.command("canvas")(self.canvas)
+        self.command("charts")(self.charts)
         # Uncomment when infield v2 config migration is ready
         # self.command("infield-configs")(self.infield_configs)
 
@@ -694,6 +700,149 @@ class MigrateApp(typer.Typer):
             )
         )
 
+    @classmethod
+    def annotations(
+        cls,
+        ctx: typer.Context,
+        mapping_file: Annotated[
+            Path | None,
+            typer.Option(
+                "--mapping-file",
+                "-m",
+                help="Path to the mapping file that contains the mapping from Annotations to CogniteDiagramAnnotation. "
+                "This file is expected to have the following columns: [id, space, externalId, ingestionView].",
+            ),
+        ] = None,
+        data_set_id: Annotated[
+            str | None,
+            typer.Option(
+                "--data-set-id",
+                "-s",
+                help="The data set ID to select for the annotations to migrate. If not provided and the mapping file is not provided, "
+                "an interactive selection will be performed to select the data set to migrate annotations from.",
+            ),
+        ] = None,
+        instance_space: Annotated[
+            str | None,
+            typer.Option(
+                "--instance-space",
+                "-i",
+                help="The instance space to use for the migrated annotations. Required when using --data-set-id.",
+            ),
+        ] = None,
+        asset_annotation_mapping: Annotated[
+            str | None,
+            typer.Option(
+                "--asset-annotation-mapping",
+                "-a",
+                help="The ingestion mapping to use for asset-linked annotations. If not provided, "
+                "the default mapping (cdf_asset_annotations_mapping) will be used.",
+            ),
+        ] = None,
+        file_annotation_mapping: Annotated[
+            str | None,
+            typer.Option(
+                "--file-annotation-mapping",
+                "-f",
+                help="The ingestion mapping to use for file-linked annotations. If not provided, "
+                "the default mapping (cdf_file_annotations_mapping) will be used.",
+            ),
+        ] = None,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where logs will be stored. If the directory does not exist, it will be created.",
+            ),
+        ] = Path(f"migration_logs_{TODAY!s}"),
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the migration will not be executed, but only a report of what would be done is printed.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Migrate Annotations to CogniteDiagramAnnotation edges in data modeling.
+
+        Annotations are diagram annotations that link assets or files to other resources. This command
+        migrates them to edges in the data modeling space, preserving the relationships and metadata.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+
+        if data_set_id is not None and mapping_file is not None:
+            raise typer.BadParameter("Cannot specify both data_set_id and mapping_file")
+        elif mapping_file is not None:
+            selected: AssetCentricMigrationSelector = MigrationCSVFileSelector(
+                datafile=mapping_file, kind="Annotations"
+            )
+            annotation_io = AnnotationMigrationIO(client)
+        elif data_set_id is not None:
+            if instance_space is None:
+                raise typer.BadParameter("--instance-space is required when using --data-set-id")
+            selected = MigrateDataSetSelector(data_set_external_id=data_set_id, kind="Annotations")
+            annotation_io = AnnotationMigrationIO(
+                client,
+                instance_space=instance_space,
+                default_asset_annotation_mapping=asset_annotation_mapping,
+                default_file_annotation_mapping=file_annotation_mapping,
+            )
+        else:
+            # Interactive selection
+            selector = FileMetadataInteractiveSelect(client, "migrate")
+            selected_data_set_id = selector.select_data_set(allow_empty=False)
+            dm_selector = DataModelingSelect(client, "migrate")
+            selected_instance_space = dm_selector.select_instance_space(
+                multiselect=False,
+                message="In which instance space do you want to create the annotations?",
+                include_empty=True,
+            )
+            if selected_instance_space is None:
+                raise typer.Abort()
+            asset_annotations_selector = ResourceViewMappingInteractiveSelect(client, "migrate asset annotations")
+            asset_annotation_mapping = asset_annotations_selector.select_resource_view_mapping(
+                resource_type="assetAnnotation",
+            ).external_id
+            file_annotations_selector = ResourceViewMappingInteractiveSelect(client, "migrate file annotations")
+            file_annotation_mapping = file_annotations_selector.select_resource_view_mapping(
+                resource_type="fileAnnotation",
+            ).external_id
+
+            selected = MigrateDataSetSelector(data_set_external_id=selected_data_set_id, kind="Annotations")
+            annotation_io = AnnotationMigrationIO(
+                client,
+                instance_space=selected_instance_space,
+                default_asset_annotation_mapping=asset_annotation_mapping,
+                default_file_annotation_mapping=file_annotation_mapping,
+            )
+
+            dry_run = questionary.confirm("Do you want to perform a dry run?", default=dry_run).ask()
+            verbose = questionary.confirm("Do you want verbose output?", default=verbose).ask()
+            if any(res is None for res in [dry_run, verbose]):
+                raise typer.Abort()
+
+        cmd = MigrationCommand()
+        cmd.run(
+            lambda: cmd.migrate(
+                selected=selected,
+                data=annotation_io,
+                mapper=AssetCentricMapper[Annotation](client),
+                log_dir=log_dir,
+                dry_run=dry_run,
+                verbose=verbose,
+            )
+        )
+
     @staticmethod
     def canvas(
         ctx: typer.Context,
@@ -704,6 +853,23 @@ class MigrateApp(typer.Typer):
                 "performed to select the Canvas to migrate."
             ),
         ] = None,
+        skip_on_missing_ref: Annotated[
+            bool,
+            typer.Option(
+                "--skip-on-missing-ref",
+                "-s",
+                help="If set, the migration will skip Canvases that reference resources that have not been migrated to data modeling. "
+                "If not set, the migration will continue but the result will exclude the missing references.",
+            ),
+        ] = False,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where migration logs will be stored.",
+            ),
+        ] = Path(f"migration_logs_{TODAY}"),
         dry_run: Annotated[
             bool,
             typer.Option(
@@ -729,12 +895,86 @@ class MigrateApp(typer.Typer):
         is populated with the mapping from Asset-Centric resources to the new data modeling resources.
         """
         client = EnvironmentVariables.create_from_environment().get_client()
+        if external_id is None:
+            interactive = InteractiveCanvasSelect(client)
+            external_id = interactive.select_external_ids()
+            log_dir = questionary.path("Specify log directory for migration logs:", default=str(log_dir)).ask()
+            dry_run = questionary.confirm("Do you want to perform a dry run?", default=dry_run).ask()
+            verbose = questionary.confirm("Do you want verbose output?", default=verbose).ask()
+            if any(res is None for res in [log_dir, dry_run, verbose]):
+                raise typer.Abort()
+            log_dir = Path(log_dir)
 
-        cmd = MigrationCanvasCommand()
+        cmd = MigrationCommand()
+        selector = CanvasExternalIdSelector(external_ids=tuple(external_id))
         cmd.run(
-            lambda: cmd.migrate_canvas(
-                client,
-                external_ids=external_id,
+            lambda: cmd.migrate(
+                selected=selector,
+                data=CanvasIO(client, exclude_existing_version=True),
+                mapper=CanvasMapper(client, dry_run=dry_run, skip_on_missing_ref=skip_on_missing_ref),
+                log_dir=log_dir,
+                dry_run=dry_run,
+                verbose=verbose,
+            )
+        )
+
+    @staticmethod
+    def charts(
+        ctx: typer.Context,
+        external_id: Annotated[
+            list[str] | None,
+            typer.Argument(
+                help="The external ID of the Chart to migrate. If not provided, an interactive selection will be "
+                "performed to select the Charts to migrate."
+            ),
+        ] = None,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where migration logs will be stored.",
+            ),
+        ] = Path(f"migration_logs_{TODAY}"),
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the migration will not be executed, but only a report of "
+                "what would be done is printed. This is useful for checking that all time series referenced by the Charts "
+                "have been migrated to the new data modeling resources in CDF.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Migrate Charts from time series references to data modeling in CDF.
+
+        This command expects that the CogniteMigration data model is already deployed, and that the Mapping view
+        is populated with the mapping from time series to the new data modeling resources.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+
+        selected_external_ids: list[str]
+        if external_id:
+            selected_external_ids = external_id
+        else:
+            selected_external_ids = InteractiveChartSelect(client).select_external_ids()
+
+        cmd = MigrationCommand()
+        cmd.run(
+            lambda: cmd.migrate(
+                selected=ChartExternalIdSelector(external_ids=tuple(selected_external_ids)),
+                data=ChartIO(client),
+                mapper=ChartMapper(client),
+                log_dir=log_dir,
                 dry_run=dry_run,
                 verbose=verbose,
             )

cognite_toolkit/_cdf_tk/apps/_modules_app.py

@@ -6,7 +6,6 @@ from rich import print
 
 from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
 from cognite_toolkit._cdf_tk.commands import ModulesCommand, PullCommand
-from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._version import __version__
 
@@ -58,7 +57,6 @@ class ModulesApp(typer.Typer):
                 "--library-url",
                 "-u",
                 help="URL of the library to add to the project.",
-                hidden=not Flags.EXTERNAL_LIBRARIES.is_enabled(),
             ),
         ] = None,
         library_checksum: Annotated[
@@ -67,7 +65,6 @@ class ModulesApp(typer.Typer):
                 "--library-checksum",
                 "-c",
                 help="Checksum of the library to add to the project.",
-                hidden=not Flags.EXTERNAL_LIBRARIES.is_enabled(),
             ),
         ] = None,
         verbose: Annotated[