cognite-toolkit 0.5.97__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/apps/_modules_app.py

@@ -51,6 +51,22 @@ class ModulesApp(typer.Typer):
                 help="Clean target directory if it exists",
             ),
         ] = False,
+        library_url: Annotated[
+            str | None,
+            typer.Option(
+                "--library-url",
+                "-u",
+                help="URL of the library to add to the project.",
+            ),
+        ] = None,
+        library_checksum: Annotated[
+            str | None,
+            typer.Option(
+                "--library-checksum",
+                "-c",
+                help="Checksum of the library to add to the project.",
+            ),
+        ] = None,
         verbose: Annotated[
             bool,
             typer.Option(
@@ -62,12 +78,20 @@ class ModulesApp(typer.Typer):
     ) -> None:
         """Initialize or upgrade a new CDF project with templates interactively."""
 
+        if library_url and not library_checksum:
+            raise typer.BadParameter(
+                "--library-checksum must be provided when --library-url is specified.",
+                param_hint="--library-checksum",
+            )
+
         with ModulesCommand() as cmd:
             cmd.run(
                 lambda: cmd.init(
                     organization_dir=organization_dir,
                     select_all=all,
                     clean=clean,
+                    library_url=library_url,
+                    library_checksum=library_checksum,
                 )
             )
 

cognite_toolkit/_cdf_tk/apps/_profile_app.py

@@ -58,15 +58,24 @@ class ProfileApp(typer.Typer):
         cmd = ProfileAssetCommand(output_spreadsheet)
         cmd.run(
             lambda: cmd.assets(
-                client,
-                hierarchy,
-                verbose,
+                client=client,
+                hierarchy=hierarchy,
+                verbose=verbose,
             )
         )
 
     @staticmethod
     def asset_centric(
         ctx: typer.Context,
+        hierarchy: Annotated[
+            str | None,
+            typer.Option(
+                "--hierarchy",
+                "-h",
+                help="The asset hierarchy to profile. This should be the externalId of the root asset. If not provided,"
+                " an interactive prompt will be used to select the hierarchy (or select all assets-centric assets).",
+            ),
+        ] = None,
         output_spreadsheet: Annotated[
             Path | None,
             typer.Option(
@@ -85,6 +94,7 @@ class ProfileApp(typer.Typer):
         cmd.run(
             lambda: cmd.asset_centric(
                 client,
+                hierarchy,
                 verbose,
             )
         )

cognite_toolkit/_cdf_tk/apps/_purge.py

@@ -1,11 +1,27 @@
-from typing import Annotated, Any
+from enum import Enum
+from pathlib import Path
+from typing import Annotated, Any, cast
 
+import questionary
 import typer
 from rich import print
 
 from cognite_toolkit._cdf_tk.commands import PurgeCommand
-from cognite_toolkit._cdf_tk.feature_flags import Flags
+from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
+from cognite_toolkit._cdf_tk.storageio.selectors import (
+    InstanceFileSelector,
+    InstanceSelector,
+    InstanceViewSelector,
+    SelectedView,
+)
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
+from cognite_toolkit._cdf_tk.utils.cli_args import parse_view_str
+from cognite_toolkit._cdf_tk.utils.interactive_select import AssetInteractiveSelect, DataModelingSelect
+
+
+class InstanceTypeEnum(str, Enum):
+    node = "node"
+    edge = "edge"
 
 
 class PurgeApp(typer.Typer):
@@ -14,16 +30,15 @@ class PurgeApp(typer.Typer):
         self.callback(invoke_without_command=True)(self.main)
         self.command("dataset")(self.purge_dataset)
         self.command("space")(self.purge_space)
-        if Flags.PURGE_INSTANCES.is_enabled():
-            self.command("instances")(self.purge_instances)
+        self.command("instances")(self.purge_instances)
 
     def main(self, ctx: typer.Context) -> None:
-        """Commands purge functionality"""
+        """Commands deleting data from Cognite Data Fusion."""
         if ctx.invoked_subcommand is None:
             print("Use [bold yellow]cdf purge --help[/] for more information.")
 
+    @staticmethod
     def purge_dataset(
-        self,
         ctx: typer.Context,
         external_id: Annotated[
             str | None,
@@ -31,12 +46,38 @@ class PurgeApp(typer.Typer):
                 help="External id of the dataset to purge. If not provided, interactive mode will be used.",
             ),
         ] = None,
-        include_dataset: Annotated[
+        archive_dataset: Annotated[
             bool,
             typer.Option(
-                "--include-dataset",
-                "-i",
-                help="Include dataset in the purge. This will also archive the dataset.",
+                "--archive-dataset",
+                help="Whether to archive the dataset itself after purging its contents.",
+            ),
+        ] = False,
+        skip_data: Annotated[
+            bool,
+            typer.Option(
+                "--skip-data",
+                "-s",
+                help="Skip deleting the data in the dataset, only delete configurations. The resources that are "
+                "considered data are: time series, event, files, assets, sequences, relationships, "
+                "labels, and 3D Models",
+            ),
+        ] = False,
+        include_configurations: Annotated[
+            bool,
+            typer.Option(
+                "--include-configurations",
+                "-c",
+                help="Include configurations, workflows, extraction pipelines and transformations in the purge.",
+            ),
+        ] = False,
+        asset_recursive: Annotated[
+            bool,
+            typer.Option(
+                "--asset-recursive",
+                "-a",
+                help="When deleting assets, delete all child assets recursively. CAVEAT: This can lead to assets"
+                " not in the selected dataset being deleted if they are children of assets in the dataset.",
             ),
         ] = False,
         dry_run: Annotated[
@@ -67,19 +108,49 @@ class PurgeApp(typer.Typer):
         """This command will delete the contents of the specified dataset"""
         cmd = PurgeCommand()
         client = EnvironmentVariables.create_from_environment().get_client()
+
+        if external_id is None:
+            # Is Interactive
+            interactive = AssetInteractiveSelect(client, operation="purge")
+            external_id = interactive.select_data_set(allow_empty=False)
+            skip_data = not questionary.confirm(
+                "Delete data in the dataset (time series, events, files, assets, sequences, relationships, labels, 3D models)?",
+                default=True,
+            ).ask()
+            include_configurations = questionary.confirm(
+                "Delete configurations (workflows, extraction pipelines and transformations) in the dataset?",
+                default=False,
+            ).ask()
+            asset_recursive = questionary.confirm(
+                "When deleting assets, delete all child assets recursively? (WARNING: This can lead "
+                "to assets not in the selected dataset being deleted if they are children of assets in the dataset.)",
+                default=False,
+            ).ask()
+            archive_dataset = questionary.confirm("Archive the dataset itself after purging?", default=False).ask()
+            dry_run = questionary.confirm("Dry run?", default=True).ask()
+            verbose = questionary.confirm("Verbose?", default=True).ask()
+
+            user_options = [archive_dataset, dry_run, verbose, skip_data, include_configurations, asset_recursive]
+
+            if any(selected is None for selected in user_options):
+                raise typer.Abort("Aborted by user.")
+
         cmd.run(
             lambda: cmd.dataset(
                 client,
                 external_id,
-                include_dataset,
+                archive_dataset,
+                not skip_data,
+                include_configurations,
+                asset_recursive,
                 dry_run,
                 auto_yes,
                 verbose,
             )
         )
 
+    @staticmethod
     def purge_space(
-        self,
         ctx: typer.Context,
         space: Annotated[
             str | None,
@@ -95,6 +166,20 @@ class PurgeApp(typer.Typer):
                 help="Include space in the purge. This will also delete the space.",
             ),
         ] = False,
+        delete_datapoints: Annotated[
+            bool,
+            typer.Option(
+                "--delete-datapoints",
+                help="Delete datapoints linked to CogniteTimeSeries nodes in the space.",
+            ),
+        ] = False,
+        delete_file_content: Annotated[
+            bool,
+            typer.Option(
+                "--delete-file-content",
+                help="Delete file content linked to CogniteFile nodes in the space.",
+            ),
+        ] = False,
         dry_run: Annotated[
             bool,
             typer.Option(
@@ -124,25 +209,47 @@ class PurgeApp(typer.Typer):
 
         cmd = PurgeCommand()
         client = EnvironmentVariables.create_from_environment().get_client()
+
+        if space is None:
+            # Is Interactive
+            interactive = DataModelingSelect(client, operation="purge")
+            space_type = interactive.select_space_type()
+            if space_type == "empty":
+                space = interactive.select_empty_spaces(multiselect=False)
+            elif space_type == "instance":
+                space = interactive.select_instance_space(multiselect=False)
+            elif space_type == "schema":
+                space = interactive.select_schema_space(include_global=False).space
+            else:
+                raise ToolkitValueError("Invalid space type selected.")
+            dry_run = questionary.confirm("Dry run?", default=True).ask()
+            if space_type == "empty":
+                include_space = True
+            else:
+                include_space = questionary.confirm("Delete the space itself?", default=False).ask()
+
         cmd.run(
             lambda: cmd.space(
-                client,
-                space,
-                include_space,
-                dry_run,
-                auto_yes,
-                verbose,
+                client=client,
+                selected_space=space,
+                include_space=include_space,
+                delete_datapoints=delete_datapoints,
+                delete_file_content=delete_file_content,
+                dry_run=dry_run,
+                auto_yes=auto_yes,
+                verbose=verbose,
             )
         )
 
     @staticmethod
     def purge_instances(
         view: Annotated[
-            list[str] | None,
+            str | None,
             typer.Argument(
                 help="Purge instances with properties in the specified view. Expected format is "
-                "'space externalId version'. For example 'cdf_cdm CogniteTimeSeries v1' will purge all nodes"
-                "that have properties in the CogniteTimeSeries view. If not provided, interactive mode will be used.",
+                "'space:externalId/version'. For example 'cdf_cdm:CogniteTimeSeries/v1' will purge all nodes"
+                "that have properties in the CogniteTimeSeries view. If not provided and no "
+                "instance list is provided, interactive mode will be used.",
             ),
         ] = None,
         instance_space: Annotated[
@@ -154,7 +261,7 @@ class PurgeApp(typer.Typer):
             ),
         ] = None,
         instance_type: Annotated[
-            str,
+            InstanceTypeEnum,
             typer.Option(
                 "--instance-type",
                 "-t",
@@ -162,7 +269,22 @@ class PurgeApp(typer.Typer):
                 case_sensitive=False,
                 show_default=True,
             ),
-        ] = "node",
+        ] = InstanceTypeEnum.node,
+        instance_list: Annotated[
+            Path | None,
+            typer.Option(
+                "--list",
+                "-l",
+                help="Path to a file containing a list of instance external IDs to purge. This file should be a csv file with space,externalId,instanceType columns."
+                "If this option is provided, the view, instance-space and instance-type options will be ignored.",
+                exists=True,
+                file_okay=True,
+                dir_okay=False,
+                readable=True,
+                resolve_path=True,
+                show_default=True,
+            ),
+        ] = None,
         dry_run: Annotated[
             bool,
             typer.Option(
@@ -171,6 +293,17 @@ class PurgeApp(typer.Typer):
                 help="Whether to do a dry-run, do dry-run if present.",
             ),
         ] = False,
+        unlink: Annotated[
+            bool,
+            typer.Option(
+                "--skip-unlink",
+                "-u",
+                help="This only applies to CogniteTimeSeries and CogniteFile nodes. By default, the purge command will unlink the "
+                "node from the datapoints/file content before deleting the node. If you want to delete the nodes with their datapoints/file content, "
+                "you can skip the unlinking. Note that this will delete the datapoints/file content "
+                "themselves, not the links to their parent nodes.",
+            ),
+        ] = True,
         auto_yes: Annotated[
             bool,
             typer.Option(
@@ -192,12 +325,40 @@ class PurgeApp(typer.Typer):
 
         cmd = PurgeCommand()
         client = EnvironmentVariables.create_from_environment().get_client(enable_set_pending_ids=True)
+        is_interactive = view is None and instance_list is None
+        selector: InstanceSelector
+        if is_interactive:
+            interactive = DataModelingSelect(client, operation="purge")
+            select_view = interactive.select_view(include_global=True)
+            selected_instance_type = interactive.select_instance_type(select_view.used_for)
+            instance_space = interactive.select_instance_space(True, select_view.as_id(), selected_instance_type)
+            selector = InstanceViewSelector(
+                view=SelectedView(
+                    space=select_view.space, external_id=select_view.external_id, version=select_view.version
+                ),
+                instance_type=selected_instance_type,
+                instance_spaces=tuple(instance_space) if instance_space else None,
+            )
+            dry_run = questionary.confirm("Dry run?", default=True).ask()
+            unlink = questionary.confirm("Unlink instances connected to timeseries or files?", default=True).ask()
+        elif instance_list is not None:
+            selector = InstanceFileSelector(datafile=instance_list)
+        elif view is not None:
+            view_id = parse_view_str(view)
+            selector = InstanceViewSelector(
+                view=SelectedView(
+                    space=view_id.space, external_id=view_id.external_id, version=cast(str, view_id.version)
+                ),
+                instance_type=instance_type.value,
+                instance_spaces=tuple(instance_space) if instance_space is not None else None,
+            )
+        else:
+            raise ToolkitValueError("Invalid combination of arguments.")
         cmd.run(
             lambda: cmd.instances(
-                client,
-                view,
-                instance_space,
-                instance_type,
+                client=client,
+                selector=selector,
+                unlink=unlink,
                 dry_run=dry_run,
                 auto_yes=auto_yes,
                 verbose=verbose,

cognite_toolkit/_cdf_tk/apps/_run.py

@@ -23,9 +23,15 @@ class RunApp(typer.Typer):
         self.command("workflow")(self.run_workflow)
         self.add_typer(RunFunctionApp(*args, **kwargs), name="function")
 
+    @staticmethod
+    def _print_deprecation_warning() -> None:
+        """Print deprecation warning for the run plugin."""
+        print("The run plugin is deprecated and will be replaced by the dev plugin in v0.8.0.")
+
     @staticmethod
     def main(ctx: typer.Context) -> None:
         """Commands to execute processes in CDF."""
+        RunApp._print_deprecation_warning()
         if ctx.invoked_subcommand is None:
             print("Use [bold yellow]cdf run --help[/] for more information.")
 
@@ -51,6 +57,7 @@ class RunApp(typer.Typer):
         ] = False,
     ) -> None:
         """This command will run the specified transformation using a one-time session."""
+        RunApp._print_deprecation_warning()
         cmd = RunTransformationCommand()
         client = EnvironmentVariables.create_from_environment().get_client()
         cmd.run(lambda: cmd.run_transformation(client, external_id))
@@ -108,6 +115,7 @@ class RunApp(typer.Typer):
         ] = False,
     ) -> None:
         """This command will run the specified workflow."""
+        RunApp._print_deprecation_warning()
         cmd = RunWorkflowCommand()
         env_vars = EnvironmentVariables.create_from_environment()
         cmd.run(lambda: cmd.run_workflow(env_vars, organization_dir, env_name, external_id, version, wait))
@@ -123,6 +131,7 @@ class RunFunctionApp(typer.Typer):
     @staticmethod
     def main(ctx: typer.Context) -> None:
         """Commands to execute function."""
+        RunApp._print_deprecation_warning()
         if ctx.invoked_subcommand is None:
             print("Use [bold yellow]cdf run function --help[/] for more information.")
 
@@ -178,6 +187,7 @@ class RunFunctionApp(typer.Typer):
         ] = False,
     ) -> None:
         """This command will run the specified function locally."""
+        RunApp._print_deprecation_warning()
         cmd = RunFunctionCommand()
         env_vars = EnvironmentVariables.create_from_environment()
         cmd.run(
@@ -243,6 +253,7 @@ class RunFunctionApp(typer.Typer):
         ] = False,
     ) -> None:
         """This command will run the specified function (assuming it is deployed) in CDF."""
+        RunApp._print_deprecation_warning()
         cmd = RunFunctionCommand()
         env_vars = EnvironmentVariables.create_from_environment()
         cmd.run(lambda: cmd.run_cdf(env_vars, organization_dir, env_name, external_id, schedule, wait))

cognite_toolkit/_cdf_tk/apps/_upload_app.py

@@ -0,0 +1,107 @@
+from pathlib import Path
+from typing import Annotated, Any
+
+import questionary
+import typer
+from questionary import Choice
+
+from cognite_toolkit._cdf_tk.commands import UploadCommand
+from cognite_toolkit._cdf_tk.constants import DATA_DEFAULT_DIR, DATA_MANIFEST_SUFFIX, DATA_RESOURCE_DIR
+from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
+
+DEFAULT_INPUT_DIR = Path.cwd() / DATA_DEFAULT_DIR
+
+
+class UploadApp(typer.Typer):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.callback(invoke_without_command=True)(self.upload_main)
+        self.command("dir")(self.upload_dir)
+
+    @staticmethod
+    def upload_main(ctx: typer.Context) -> None:
+        """Commands to upload data to CDF."""
+        if ctx.invoked_subcommand is None:
+            print("Use [bold yellow]cdf upload --help[/] for more information.")
+        return None
+
+    @staticmethod
+    def upload_dir(
+        ctx: typer.Context,
+        input_dir: Annotated[
+            Path | None,
+            typer.Argument(
+                help="The directory containing the data to upload. If not specified, an interactive prompt will ask for the directory.",
+                exists=True,
+                file_okay=False,
+                dir_okay=True,
+                resolve_path=True,
+            ),
+        ] = None,
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the command will not actually upload the data, but will print what would be uploaded.",
+            ),
+        ] = False,
+        deploy_resources: Annotated[
+            bool,
+            typer.Option(
+                "--deploy-resources",
+                "-r",
+                help="If set, the command will look for resource configuration files in adjacent folders and create them if they do not exist.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Commands to upload data to CDF."""
+        cmd = UploadCommand()
+        if input_dir is None:
+            input_candidate = sorted({p.parent for p in DEFAULT_INPUT_DIR.rglob(f"**/*{DATA_MANIFEST_SUFFIX}")})
+            if not input_candidate:
+                typer.echo(f"No data manifests found in default directory: {DEFAULT_INPUT_DIR}")
+                raise typer.Exit(code=1)
+            input_dir = questionary.select(
+                "Select the input directory containing the data to upload:",
+                choices=[Choice(str(option.name), value=option) for option in input_candidate],
+            ).ask()
+            if input_dir is None:
+                typer.echo("No input directory selected. Exiting.")
+                raise typer.Exit(code=1)
+            dry_run = questionary.confirm("Proceed with dry run?", default=dry_run).ask()
+            if dry_run is None:
+                typer.echo("No selection made for dry run. Exiting.")
+                raise typer.Exit(code=1)
+            resource_dir = Path(input_dir) / DATA_RESOURCE_DIR
+            if resource_dir.exists():
+                if resource_dir.is_relative_to(Path.cwd()):
+                    display_name = resource_dir.relative_to(Path.cwd()).as_posix()
+                else:
+                    display_name = resource_dir.as_posix()
+
+                deploy_resources = questionary.confirm(
+                    f"Deploy resources found in {display_name!r}?", default=deploy_resources
+                ).ask()
+                if deploy_resources is None:
+                    typer.echo("No selection made for deploying resources. Exiting.")
+                    raise typer.Exit(code=1)
+
+        client = EnvironmentVariables.create_from_environment().get_client()
+        cmd.run(
+            lambda: cmd.upload(
+                input_dir=input_dir,
+                dry_run=dry_run,
+                verbose=verbose,
+                deploy_resources=deploy_resources,
+                client=client,
+            )
+        )

cognite_toolkit/_cdf_tk/builders/__init__.py

@@ -3,7 +3,7 @@ from pathlib import Path
 
 from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning
 
-from ._base import Builder, DefaultBuilder, get_loader
+from ._base import Builder, DefaultBuilder, get_resource_crud
 from ._datamodels import DataModelBuilder
 from ._file import FileBuilder
 from ._function import FunctionBuilder
@@ -36,5 +36,5 @@ __all__ = [
     "StreamlitBuilder",
     "TransformationBuilder",
     "create_builder",
-    "get_loader",
+    "get_resource_crud",
 ]