cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/apps/_modules_app.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Annotated, Optional
+from typing import Annotated
 
 import typer
 from rich import print
@@ -30,7 +30,7 @@ class ModulesApp(typer.Typer):
     def init(
         self,
         organization_dir: Annotated[
-            Optional[Path],
+            Path | None,
             typer.Argument(
                 help="Directory path to project to initialize or upgrade with templates.",
             ),
@@ -51,6 +51,22 @@ class ModulesApp(typer.Typer):
                 help="Clean target directory if it exists",
             ),
         ] = False,
+        library_url: Annotated[
+            str | None,
+            typer.Option(
+                "--library-url",
+                "-u",
+                help="URL of the library to add to the project.",
+            ),
+        ] = None,
+        library_checksum: Annotated[
+            str | None,
+            typer.Option(
+                "--library-checksum",
+                "-c",
+                help="Checksum of the library to add to the project.",
+            ),
+        ] = None,
         verbose: Annotated[
             bool,
             typer.Option(
@@ -62,14 +78,22 @@ class ModulesApp(typer.Typer):
     ) -> None:
         """Initialize or upgrade a new CDF project with templates interactively."""
 
-        cmd = ModulesCommand()
-        cmd.run(
-            lambda: cmd.init(
-                organization_dir=organization_dir,
-                select_all=all,
-                clean=clean,
+        if library_url and not library_checksum:
+            raise typer.BadParameter(
+                "--library-checksum must be provided when --library-url is specified.",
+                param_hint="--library-checksum",
+            )
+
+        with ModulesCommand() as cmd:
+            cmd.run(
+                lambda: cmd.init(
+                    organization_dir=organization_dir,
+                    select_all=all,
+                    clean=clean,
+                    library_url=library_url,
+                    library_checksum=library_checksum,
+                )
             )
-        )
 
     def upgrade(
         self,
@@ -116,14 +140,14 @@ class ModulesApp(typer.Typer):
         ] = False,
     ) -> None:
         """Add one or more new module(s) to the project."""
-        cmd = ModulesCommand()
-        cmd.run(lambda: cmd.add(organization_dir=organization_dir))
+        with ModulesCommand() as cmd:
+            cmd.run(lambda: cmd.add(organization_dir=organization_dir))
 
     def pull(
         self,
         ctx: typer.Context,
         module_name_or_path: Annotated[
-            Optional[str],
+            str | None,
             typer.Argument(
                 help="The module or path to module to pull from CDF. If not provided, interactive mode will be used.",
                 allow_dash=True,
@@ -187,7 +211,7 @@ class ModulesApp(typer.Typer):
             ),
         ] = CDF_TOML.cdf.default_organization_dir,
         build_env: Annotated[
-            Optional[str],
+            str | None,
             typer.Option(
                 "--env",
                 help="Build environment to use.",

cognite_toolkit/_cdf_tk/apps/_profile_app.py

@@ -1,9 +1,15 @@
-from typing import Any
+from pathlib import Path
+from typing import Annotated, Any
 
 import typer
 from rich import print
 
-from cognite_toolkit._cdf_tk.commands import ProfileCommand
+from cognite_toolkit._cdf_tk.commands import (
+    ProfileAssetCentricCommand,
+    ProfileAssetCommand,
+    ProfileRawCommand,
+    ProfileTransformationCommand,
+)
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 
 
@@ -11,26 +17,159 @@ class ProfileApp(typer.Typer):
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.callback(invoke_without_command=True)(self.main)
+        self.command("assets")(self.assets)
         self.command("asset-centric")(self.asset_centric)
+        self.command("transformations")(self.transformations)
+        self.command("raw")(self.raw)
 
     def main(self, ctx: typer.Context) -> None:
         """Commands profile functionality"""
         if ctx.invoked_subcommand is None:
             print("Use [bold yellow]cdf profile --help[/] for more information.")
 
+    @staticmethod
+    def assets(
+        ctx: typer.Context,
+        hierarchy: Annotated[
+            str | None,
+            typer.Option(
+                "--hierarchy",
+                "-h",
+                help="The asset hierarchy to profile. This should be the externalId of the root asset. If not provided,"
+                " an interactive prompt will be used to select the hierarchy.",
+            ),
+        ] = None,
+        output_spreadsheet: Annotated[
+            Path | None,
+            typer.Option(
+                "--output-spreadsheet",
+                "-o",
+                help="The path to the output spreadsheet. If not provided, the output will only be printed to the console.",
+            ),
+        ] = None,
+        verbose: bool = False,
+    ) -> None:
+        """This command gives an overview over the assets in the given hierarchy.
+        It works by listing all assets, events, files, timeseries, and sequences related to the given hierarchy.
+        In addition, it lists the data sets that is used for each of the resources, the transformations that writes to
+        these data sets, and the RAW tables that is used in these transformations.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+        cmd = ProfileAssetCommand(output_spreadsheet)
+        cmd.run(
+            lambda: cmd.assets(
+                client=client,
+                hierarchy=hierarchy,
+                verbose=verbose,
+            )
+        )
+
     @staticmethod
     def asset_centric(
         ctx: typer.Context,
+        hierarchy: Annotated[
+            str | None,
+            typer.Option(
+                "--hierarchy",
+                "-h",
+                help="The asset hierarchy to profile. This should be the externalId of the root asset. If not provided,"
+                " an interactive prompt will be used to select the hierarchy (or select all assets-centric assets).",
+            ),
+        ] = None,
+        output_spreadsheet: Annotated[
+            Path | None,
+            typer.Option(
+                "--output-spreadsheet",
+                "-o",
+                help="The path to the output spreadsheet. If not provided, the output will only be printed to the console.",
+            ),
+        ] = None,
         verbose: bool = False,
     ) -> None:
         """This command gives an overview over the metadata and labels for each of the asset-centric resources.
         This shows an approximation of unstructured data count. This can, for example, be used to estimate the
         effort to model this data in data modeling."""
         client = EnvironmentVariables.create_from_environment().get_client()
-        cmd = ProfileCommand()
+        cmd = ProfileAssetCentricCommand(output_spreadsheet)
         cmd.run(
             lambda: cmd.asset_centric(
                 client,
+                hierarchy,
+                verbose,
+            )
+        )
+
+    @staticmethod
+    def transformations(
+        destination: Annotated[
+            str | None,
+            typer.Option(
+                "--destination",
+                "-d",
+                help="Destination type the transformations data should be written to. This can be 'assets', 'events', 'files',"
+                "'timeseries', or 'sequences'. If not provided, and interactive mode is enabled, the user will be prompted to select a destination.",
+            ),
+        ] = None,
+        output_spreadsheet: Annotated[
+            Path | None,
+            typer.Option(
+                "--output-spreadsheet",
+                "-o",
+                help="The path to the output spreadsheet. If not provided, the output will only be printed to the console.",
+            ),
+        ] = None,
+        verbose: bool = False,
+    ) -> None:
+        """This command gives an overview over the transformations that write to the given destination.
+        It works by checking all transformations that writes to the given destination, lists the sources of the data,
+        and the target columns.
+        This is intended to show the flow of data from raw into CDF. This can, for example, be used to determine the
+        source of the data in a specific CDF resource.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+        cmd = ProfileTransformationCommand(output_spreadsheet)
+        cmd.run(
+            lambda: cmd.transformation(
+                client,
+                destination,
+                verbose,
+            )
+        )
+
+    @staticmethod
+    def raw(
+        ctx: typer.Context,
+        destination: Annotated[
+            str | None,
+            typer.Option(
+                "--destination",
+                "-d",
+                help="Destination type the raw data should be written to. This can be 'assets', 'events', 'files',"
+                "'timeseries', or 'sequences'. If not provided, and interactive mode is enabled, the user will be prompted to select a destination.",
+            ),
+        ] = None,
+        output_spreadsheet: Annotated[
+            Path | None,
+            typer.Option(
+                "--output-spreadsheet",
+                "-o",
+                help="The path to the output spreadsheet. If not provided, the output will only be printed to the console.",
+            ),
+        ] = None,
+        verbose: bool = False,
+    ) -> None:
+        """This command gives an overview over the staging tables in CDF and where they are used.
+        It works by checking all transformations that writes to the given destination and lists all the raw tables
+        that are used in those transformations.
+        This is intended to show the flow of data from raw into CDF. This can, for example, be used to determine the
+        source of the data in a specific CDF resource.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+        cmd = ProfileRawCommand(output_spreadsheet)
+        cmd.run(
+            lambda: cmd.raw(
+                client,
+                destination,
                 verbose,
             )
         )

cognite_toolkit/_cdf_tk/apps/_purge.py

@@ -1,10 +1,27 @@
-from typing import Annotated, Any, Optional
+from enum import Enum
+from pathlib import Path
+from typing import Annotated, Any, cast
 
+import questionary
 import typer
 from rich import print
 
 from cognite_toolkit._cdf_tk.commands import PurgeCommand
+from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
+from cognite_toolkit._cdf_tk.storageio.selectors import (
+    InstanceFileSelector,
+    InstanceSelector,
+    InstanceViewSelector,
+    SelectedView,
+)
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
+from cognite_toolkit._cdf_tk.utils.cli_args import parse_view_str
+from cognite_toolkit._cdf_tk.utils.interactive_select import AssetInteractiveSelect, DataModelingSelect
+
+
+class InstanceTypeEnum(str, Enum):
+    node = "node"
+    edge = "edge"
 
 
 class PurgeApp(typer.Typer):
@@ -13,27 +30,54 @@ class PurgeApp(typer.Typer):
         self.callback(invoke_without_command=True)(self.main)
         self.command("dataset")(self.purge_dataset)
         self.command("space")(self.purge_space)
+        self.command("instances")(self.purge_instances)
 
     def main(self, ctx: typer.Context) -> None:
-        """Commands purge functionality"""
+        """Commands deleting data from Cognite Data Fusion."""
         if ctx.invoked_subcommand is None:
             print("Use [bold yellow]cdf purge --help[/] for more information.")
 
+    @staticmethod
     def purge_dataset(
-        self,
         ctx: typer.Context,
         external_id: Annotated[
-            Optional[str],
+            str | None,
             typer.Argument(
                 help="External id of the dataset to purge. If not provided, interactive mode will be used.",
             ),
         ] = None,
-        include_dataset: Annotated[
+        archive_dataset: Annotated[
             bool,
             typer.Option(
-                "--include-dataset",
-                "-i",
-                help="Include dataset in the purge. This will also archive the dataset.",
+                "--archive-dataset",
+                help="Whether to archive the dataset itself after purging its contents.",
+            ),
+        ] = False,
+        skip_data: Annotated[
+            bool,
+            typer.Option(
+                "--skip-data",
+                "-s",
+                help="Skip deleting the data in the dataset, only delete configurations. The resources that are "
+                "considered data are: time series, event, files, assets, sequences, relationships, "
+                "labels, and 3D Models",
+            ),
+        ] = False,
+        include_configurations: Annotated[
+            bool,
+            typer.Option(
+                "--include-configurations",
+                "-c",
+                help="Include configurations, workflows, extraction pipelines and transformations in the purge.",
+            ),
+        ] = False,
+        asset_recursive: Annotated[
+            bool,
+            typer.Option(
+                "--asset-recursive",
+                "-a",
+                help="When deleting assets, delete all child assets recursively. CAVEAT: This can lead to assets"
+                " not in the selected dataset being deleted if they are children of assets in the dataset.",
             ),
         ] = False,
         dry_run: Annotated[
@@ -64,22 +108,52 @@ class PurgeApp(typer.Typer):
         """This command will delete the contents of the specified dataset"""
         cmd = PurgeCommand()
         client = EnvironmentVariables.create_from_environment().get_client()
+
+        if external_id is None:
+            # Is Interactive
+            interactive = AssetInteractiveSelect(client, operation="purge")
+            external_id = interactive.select_data_set(allow_empty=False)
+            skip_data = not questionary.confirm(
+                "Delete data in the dataset (time series, events, files, assets, sequences, relationships, labels, 3D models)?",
+                default=True,
+            ).ask()
+            include_configurations = questionary.confirm(
+                "Delete configurations (workflows, extraction pipelines and transformations) in the dataset?",
+                default=False,
+            ).ask()
+            asset_recursive = questionary.confirm(
+                "When deleting assets, delete all child assets recursively? (WARNING: This can lead "
+                "to assets not in the selected dataset being deleted if they are children of assets in the dataset.)",
+                default=False,
+            ).ask()
+            archive_dataset = questionary.confirm("Archive the dataset itself after purging?", default=False).ask()
+            dry_run = questionary.confirm("Dry run?", default=True).ask()
+            verbose = questionary.confirm("Verbose?", default=True).ask()
+
+            user_options = [archive_dataset, dry_run, verbose, skip_data, include_configurations, asset_recursive]
+
+            if any(selected is None for selected in user_options):
+                raise typer.Abort("Aborted by user.")
+
         cmd.run(
             lambda: cmd.dataset(
                 client,
                 external_id,
-                include_dataset,
+                archive_dataset,
+                not skip_data,
+                include_configurations,
+                asset_recursive,
                 dry_run,
                 auto_yes,
                 verbose,
             )
         )
 
+    @staticmethod
     def purge_space(
-        self,
         ctx: typer.Context,
         space: Annotated[
-            Optional[str],
+            str | None,
             typer.Argument(
                 help="Space to purge. If not provided, interactive mode will be used.",
             ),
@@ -92,6 +166,20 @@ class PurgeApp(typer.Typer):
                 help="Include space in the purge. This will also delete the space.",
             ),
         ] = False,
+        delete_datapoints: Annotated[
+            bool,
+            typer.Option(
+                "--delete-datapoints",
+                help="Delete datapoints linked to CogniteTimeSeries nodes in the space.",
+            ),
+        ] = False,
+        delete_file_content: Annotated[
+            bool,
+            typer.Option(
+                "--delete-file-content",
+                help="Delete file content linked to CogniteFile nodes in the space.",
+            ),
+        ] = False,
         dry_run: Annotated[
             bool,
             typer.Option(
@@ -121,13 +209,158 @@ class PurgeApp(typer.Typer):
 
         cmd = PurgeCommand()
         client = EnvironmentVariables.create_from_environment().get_client()
+
+        if space is None:
+            # Is Interactive
+            interactive = DataModelingSelect(client, operation="purge")
+            space_type = interactive.select_space_type()
+            if space_type == "empty":
+                space = interactive.select_empty_spaces(multiselect=False)
+            elif space_type == "instance":
+                space = interactive.select_instance_space(multiselect=False)
+            elif space_type == "schema":
+                space = interactive.select_schema_space(include_global=False).space
+            else:
+                raise ToolkitValueError("Invalid space type selected.")
+            dry_run = questionary.confirm("Dry run?", default=True).ask()
+            if space_type == "empty":
+                include_space = True
+            else:
+                include_space = questionary.confirm("Delete the space itself?", default=False).ask()
+
         cmd.run(
             lambda: cmd.space(
-                client,
-                space,
-                include_space,
-                dry_run,
-                auto_yes,
-                verbose,
+                client=client,
+                selected_space=space,
+                include_space=include_space,
+                delete_datapoints=delete_datapoints,
+                delete_file_content=delete_file_content,
+                dry_run=dry_run,
+                auto_yes=auto_yes,
+                verbose=verbose,
+            )
+        )
+
+    @staticmethod
+    def purge_instances(
+        view: Annotated[
+            str | None,
+            typer.Argument(
+                help="Purge instances with properties in the specified view. Expected format is "
+                "'space:externalId/version'. For example 'cdf_cdm:CogniteTimeSeries/v1' will purge all nodes"
+                "that have properties in the CogniteTimeSeries view. If not provided and no "
+                "instance list is provided, interactive mode will be used.",
+            ),
+        ] = None,
+        instance_space: Annotated[
+            list[str] | None,
+            typer.Option(
+                "--instance-space",
+                "-s",
+                help="Only purge instances that are in the specified instance space(s).",
+            ),
+        ] = None,
+        instance_type: Annotated[
+            InstanceTypeEnum,
+            typer.Option(
+                "--instance-type",
+                "-t",
+                help="Type of instances to purge. Can be 'node' or 'edge'. Default is 'node'.",
+                case_sensitive=False,
+                show_default=True,
+            ),
+        ] = InstanceTypeEnum.node,
+        instance_list: Annotated[
+            Path | None,
+            typer.Option(
+                "--list",
+                "-l",
+                help="Path to a file containing a list of instance external IDs to purge. This file should be a csv file with space,externalId,instanceType columns."
+                "If this option is provided, the view, instance-space and instance-type options will be ignored.",
+                exists=True,
+                file_okay=True,
+                dir_okay=False,
+                readable=True,
+                resolve_path=True,
+                show_default=True,
+            ),
+        ] = None,
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-r",
+                help="Whether to do a dry-run, do dry-run if present.",
+            ),
+        ] = False,
+        unlink: Annotated[
+            bool,
+            typer.Option(
+                "--skip-unlink",
+                "-u",
+                help="This only applies to CogniteTimeSeries and CogniteFile nodes. By default, the purge command will unlink the "
+                "node from the datapoints/file content before deleting the node. If you want to delete the nodes with their datapoints/file content, "
+                "you can skip the unlinking. Note that this will delete the datapoints/file content "
+                "themselves, not the links to their parent nodes.",
+            ),
+        ] = True,
+        auto_yes: Annotated[
+            bool,
+            typer.Option(
+                "--yes",
+                "-y",
+                help="Automatically confirm that you are sure you want to purge the instances.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """This command will delete the contents of the specified instances."""
+
+        cmd = PurgeCommand()
+        client = EnvironmentVariables.create_from_environment().get_client(enable_set_pending_ids=True)
+        is_interactive = view is None and instance_list is None
+        selector: InstanceSelector
+        if is_interactive:
+            interactive = DataModelingSelect(client, operation="purge")
+            select_view = interactive.select_view(include_global=True)
+            selected_instance_type = interactive.select_instance_type(select_view.used_for)
+            instance_space = interactive.select_instance_space(True, select_view.as_id(), selected_instance_type)
+            selector = InstanceViewSelector(
+                view=SelectedView(
+                    space=select_view.space, external_id=select_view.external_id, version=select_view.version
+                ),
+                instance_type=selected_instance_type,
+                instance_spaces=tuple(instance_space) if instance_space else None,
+            )
+            dry_run = questionary.confirm("Dry run?", default=True).ask()
+            unlink = questionary.confirm("Unlink instances connected to timeseries or files?", default=True).ask()
+        elif instance_list is not None:
+            selector = InstanceFileSelector(datafile=instance_list)
+        elif view is not None:
+            view_id = parse_view_str(view)
+            selector = InstanceViewSelector(
+                view=SelectedView(
+                    space=view_id.space, external_id=view_id.external_id, version=cast(str, view_id.version)
+                ),
+                instance_type=instance_type.value,
+                instance_spaces=tuple(instance_space) if instance_space is not None else None,
+            )
+        else:
+            raise ToolkitValueError("Invalid combination of arguments.")
+        cmd.run(
+            lambda: cmd.instances(
+                client=client,
+                selector=selector,
+                unlink=unlink,
+                dry_run=dry_run,
+                auto_yes=auto_yes,
+                verbose=verbose,
             )
         )