cognite-toolkit 0.6.39__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/auth.py

@@ -24,10 +24,13 @@ from time import sleep
 
 import questionary
 from cognite.client.data_classes.capabilities import (
+    AssetsAcl,
     Capability,
+    ExtractionConfigsAcl,
     FunctionsAcl,
     GroupsAcl,
     ProjectsAcl,
+    RelationshipsAcl,
     SessionsAcl,
 )
 from cognite.client.data_classes.iam import Group, GroupList, GroupWrite, TokenInspection
@@ -44,6 +47,7 @@ from cognite_toolkit._cdf_tk.constants import (
     TOOLKIT_DEMO_GROUP_NAME,
     TOOLKIT_SERVICE_PRINCIPAL_GROUP_NAME,
 )
+from cognite_toolkit._cdf_tk.cruds import ExtractionPipelineConfigCRUD
 from cognite_toolkit._cdf_tk.exceptions import (
     AuthenticationError,
     AuthorizationError,
@@ -93,16 +97,6 @@ class AuthCommand(ToolkitCommand):
             raise AuthenticationError(f"Unable to verify the credentials.\n{e}")
 
         print("[green]The credentials are valid.[/green]")
-        if no_verify:
-            return
-        print(
-            Panel(
-                "Running verification, 'cdf auth verify'...",
-                title="",
-                expand=False,
-            )
-        )
-        self.verify(client, dry_run)
 
     def _store_dotenv(self, env_vars: EnvironmentVariables) -> None:
         new_env_file = env_vars.create_dotenv_file()
@@ -344,7 +338,7 @@ class AuthCommand(ToolkitCommand):
             print(f"  [bold green]OK[/] - The {existing_group.name} has all the required capabilities.")
             return []
 
-        missing_capabilities = self._merge_capabilities(missing_capabilities)
+        missing_capabilities = self.merge_capabilities(missing_capabilities)
         for s in sorted(map(str, missing_capabilities)):
             self.warn(MissingCapabilityWarning(s))
 
@@ -388,7 +382,7 @@ class AuthCommand(ToolkitCommand):
                 updated_toolkit_group.capabilities or [],
                 project=client.config.project,
             )
-        adding = self._merge_capabilities(adding)
+        adding = self.merge_capabilities(adding)
         capability_str = "capabilities" if len(adding) > 1 else "capability"
         if dry_run:
             print(f"Would have updated group {updated_toolkit_group.name} with {len(adding)} new {capability_str}.")
@@ -437,16 +431,36 @@ class AuthCommand(ToolkitCommand):
     ) -> tuple[list[Capability], dict[tuple, list[str]]]:
         loaders_by_capability_tuple: dict[tuple, list[str]] = defaultdict(list)
         capability_by_id: dict[frozenset[tuple], Capability] = {}
-        for loader_cls in cruds.RESOURCE_CRUD_LIST:
-            loader = loader_cls.create_loader(client)
-            capability = loader_cls.get_required_capability(None, read_only=False)
-            capabilities = capability if isinstance(capability, list) else [capability]
+        project_type = client.project.status().this_project.data_modeling_status
+        for crud_cls in cruds.RESOURCE_CRUD_LIST:
+            crud = crud_cls.create_loader(client)
+            if crud.prerequisite_warning() is not None:
+                continue
+            if isinstance(crud, ExtractionPipelineConfigCRUD):
+                # The Extraction Pipeline Config CRUD requires special handling.
+                # The .get_required_capability is used in the DeployCommand as well. Since, there is no way to no
+                # the extraction pipeline ID or dataSetId from an ExtractionPipelineConfigWrite object, we do not
+                # check those there. If we returned the full capability, it would always have to be all scoped.
+                # That is too restrictive in the deploy command, so we return an empty list, essentially not checking
+                # anything there. Here, we want to add the all scoped capability, so that the Toolkit group gets the
+                # correct capability.
+                capabilities: list[Capability] = [
+                    ExtractionConfigsAcl(
+                        [ExtractionConfigsAcl.Action.Read, ExtractionConfigsAcl.Action.Write],
+                        ExtractionConfigsAcl.Scope.All(),
+                    )
+                ]
+            else:
+                capability = crud_cls.get_required_capability(None, read_only=False)
+                capabilities = capability if isinstance(capability, list) else [capability]
             for cap in capabilities:
+                if project_type == "DATA_MODELING_ONLY" and isinstance(cap, AssetsAcl | RelationshipsAcl):
+                    continue
                 id_ = frozenset(cap.as_tuples())
                 if id_ not in capability_by_id:
                     capability_by_id[id_] = cap
                 for cap_tuple in cap.as_tuples():
-                    loaders_by_capability_tuple[cap_tuple].append(loader.display_name)
+                    loaders_by_capability_tuple[cap_tuple].append(crud.display_name)
         return list(capability_by_id.values()), loaders_by_capability_tuple
 
     def check_has_any_access(self, client: ToolkitClient) -> TokenInspection:
@@ -602,15 +616,21 @@ class AuthCommand(ToolkitCommand):
         return extra
 
     @staticmethod
-    def _merge_capabilities(capability_list: list[Capability]) -> list[Capability]:
+    def merge_capabilities(capability_list: list[Capability]) -> list[Capability]:
         """Merges capabilities that have the same ACL and Scope"""
-        actions_by_scope_and_cls: dict[tuple[type[Capability], Capability.Scope], set[Capability.Action]] = defaultdict(
+        actions_by_scope_and_cls: dict[tuple[type[Capability], frozenset[tuple]], set[Capability.Action]] = defaultdict(
             set
         )
         for capability in capability_list:
-            actions_by_scope_and_cls[(type(capability), capability.scope)].update(capability.actions)
+            scope_key = frozenset(capability.scope.as_tuples())
+            actions_by_scope_and_cls[(type(capability), scope_key)].update(capability.actions)
+
+        scope_map: dict[tuple[type[Capability], frozenset[tuple]], Capability.Scope] = {
+            (type(cap), frozenset(cap.scope.as_tuples())): cap.scope for cap in capability_list
+        }
+
         return [
-            cap_cls(actions=list(actions), scope=scope, allow_unknown=False)
+            cap_cls(actions=list(actions), scope=scope_map[(cap_cls, scope)], allow_unknown=False)
             for (cap_cls, scope), actions in actions_by_scope_and_cls.items()
         ]
 

cognite_toolkit/_cdf_tk/commands/build_cmd.py

@@ -20,7 +20,6 @@ from cognite_toolkit._cdf_tk.constants import (
     _RUNNING_IN_BROWSER,
     BUILD_FOLDER_ENCODING,
     DEFAULT_ENV,
-    DEV_ONLY_MODULES,
     HINT_LEAD_TEXT,
     ROOT_MODULES,
     TEMPLATE_VARS_FILE_SUFFIXES,
@@ -59,10 +58,7 @@ from cognite_toolkit._cdf_tk.data_classes import (
     SourceLocationLazy,
 )
 from cognite_toolkit._cdf_tk.exceptions import (
-    ToolkitDuplicatedModuleError,
-    ToolkitEnvError,
     ToolkitError,
-    ToolkitMissingModuleError,
     ToolkitYAMLFormatError,
 )
 from cognite_toolkit._cdf_tk.hints import Hint, ModuleDefinition, verify_module_directory
@@ -89,8 +85,8 @@ from cognite_toolkit._cdf_tk.utils.file import safe_rmtree
 from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules
 from cognite_toolkit._cdf_tk.validation import (
     validate_data_set_is_set,
+    validate_module_selection,
     validate_modules_variables,
-    validate_resource_yaml,
     validate_resource_yaml_pydantic,
 )
 from cognite_toolkit._version import __version__
@@ -143,6 +139,11 @@ class BuildCommand(ToolkitCommand):
         if selected:
             config.environment.selected = parse_user_selected_modules(selected, organization_dir)
 
+        # tracking which project the module is being built for to trace promotion
+        if client:
+            self._additional_tracking_info.project = client.config.project
+            self._additional_tracking_info.cluster = client.config.cdf_cluster
+
         directory_name = "current directory" if organization_dir == Path(".") else f"project '{organization_dir!s}'"
         root_modules = [
             module_dir for root_module in ROOT_MODULES if (module_dir := organization_dir / root_module).exists()
@@ -203,7 +204,11 @@ class BuildCommand(ToolkitCommand):
 
         user_selected_modules = config.environment.get_selected_modules(packages)
         modules = ModuleDirectories.load(organization_dir, user_selected_modules)
-        self._validate_modules(modules, config, packages, user_selected_modules, organization_dir)
+        module_warnings = validate_module_selection(modules, config, packages, user_selected_modules, organization_dir)
+        if module_warnings:
+            self.warning_list.extend(module_warnings)
+            if self.print_warning:
+                print(str(module_warnings))
 
         if verbose:
             self.console("Selected packages:")
@@ -323,9 +328,17 @@ class BuildCommand(ToolkitCommand):
                     warning_count=module_warnings,
                     status=built_status,
                     iteration=iteration,
+                    module_id=module.module_id,
+                    package_id=module.package_id,
                 )
                 build.append(built_module)
-                self.tracker.track_module_build(built_module)
+
+                if module.package_id:
+                    self._additional_tracking_info.package_ids.add(module.package_id)
+
+                if module.module_id:
+                    self._additional_tracking_info.module_ids.add(module.module_id)
+
         return build
 
     def _build_module_resources(
@@ -346,6 +359,15 @@ class BuildCommand(ToolkitCommand):
         for resource_name, resource_files in module.source_paths_by_resource_folder.items():
             source_files = self._replace_variables(resource_files, module_variables, resource_name, module.dir, verbose)
 
+            if resource_name == "data_models":
+                resource_name = "data_modeling"
+                self.warn(
+                    MediumSeverityWarning(
+                        "The resource folder 'data_models' is deprecated and will be removed in v1.0. "
+                        "Please rename the folder to 'data_modeling'."
+                    )
+                )
+
             builder = self._get_builder(build_dir, resource_name)
 
             built_resources = BuiltResourceList[Hashable]()
@@ -401,62 +423,6 @@ class BuildCommand(ToolkitCommand):
         builder = self._builder_by_resource_folder[resource_name]
         return builder
 
-    def _validate_modules(
-        self,
-        modules: ModuleDirectories,
-        config: BuildConfigYAML,
-        packages: dict[str, list[str]],
-        selected_modules: set[str | Path],
-        organization_dir: Path,
-    ) -> None:
-        # Validations: Ambiguous selection.
-        selected_names = {s for s in config.environment.selected if isinstance(s, str)}
-        if duplicate_modules := {
-            module_name: paths
-            for module_name, paths in modules.as_path_by_name().items()
-            if len(paths) > 1 and module_name in selected_names
-        }:
-            # If the user has selected a module by name, and there are multiple modules with that name, raise an error.
-            # Note, if the user uses a path to select a module, this error will not be raised.
-            raise ToolkitDuplicatedModuleError(
-                f"Ambiguous module selected in config.{config.environment.name}.yaml:", duplicate_modules
-            )
-        # Package Referenced Modules Exists
-        for package, package_modules in packages.items():
-            if package not in selected_names:
-                # We do not check packages that are not selected.
-                # Typically, the user will delete the modules that are irrelevant for them;
-                # thus we only check the selected packages.
-                continue
-            if missing_packages := set(package_modules) - modules.available_names:
-                ToolkitMissingModuleError(
-                    f"Package {package} defined in {CDFToml.file_name!s} is referring "
-                    f"the following missing modules {missing_packages}."
-                )
-
-        # Selected modules does not exists
-        if missing_modules := set(selected_modules) - modules.available:
-            hint = ModuleDefinition.long(missing_modules, organization_dir)
-            raise ToolkitMissingModuleError(
-                f"The following selected modules are missing, please check path: {missing_modules}.\n{hint}"
-            )
-
-        # Nothing is Selected
-        if not modules.selected:
-            raise ToolkitEnvError(
-                f"No selected modules specified in {config.filepath!s}, have you configured "
-                f"the environment ({config.environment.name})?"
-            )
-
-        dev_modules = modules.available_names & DEV_ONLY_MODULES
-        if dev_modules and config.environment.validation_type != "dev":
-            self.warn(
-                MediumSeverityWarning(
-                    "The following modules should [bold]only[/bold] be used a in CDF Projects designated as dev (development): "
-                    f"{humanize_collection(dev_modules)!r}",
-                )
-            )
-
     def _replace_variables(
         self,
         resource_files: Sequence[Path],
@@ -480,7 +446,7 @@ class BuildCommand(ToolkitCommand):
             # which is what we do in the deploy step to verify that the source file has not changed.
             source = SourceLocationEager(source_path, calculate_hash(source_path, shorten=True))
 
-            content = variables.replace(content, source_path.suffix)
+            content = variables.replace(content, source_path)
 
             replace_warnings = self._check_variables_replaced(content, module_dir, source_path)
 
@@ -599,8 +565,7 @@ class BuildCommand(ToolkitCommand):
     ) -> tuple[WarningList[FileReadWarning], list[tuple[Hashable, str]]]:
         warning_list = WarningList[FileReadWarning]()
 
-        if loader.yaml_cls:
-            warning_list.extend(validate_resource_yaml_pydantic(parsed, loader.yaml_cls, source.path))
+        warning_list.extend(validate_resource_yaml_pydantic(parsed, loader.yaml_cls, source.path))
 
         is_dict_item = isinstance(parsed, dict)
         items = [parsed] if isinstance(parsed, dict) else parsed
@@ -653,12 +618,6 @@ class BuildCommand(ToolkitCommand):
                 for dependency in dependencies:
                     self._dependencies_by_required[dependency].append((identifier, source.path))
 
-            api_spec = item_loader.safe_get_write_cls_parameter_spec()
-            if api_spec is not None and item_loader.yaml_cls is None:
-                # If we have a yaml cls this is already validated on the file as a whole.
-                resource_warnings = validate_resource_yaml(parsed, api_spec, source.path, element_no)
-                warning_list.extend(resource_warnings)
-
             data_set_warnings = validate_data_set_is_set(items, loader.resource_cls, source.path)
             warning_list.extend(data_set_warnings)
 

cognite_toolkit/_cdf_tk/commands/clean.py

@@ -1,8 +1,9 @@
 import traceback
+from collections.abc import Sequence
 from graphlib import TopologicalSorter
 from pathlib import Path
 
-from cognite.client.data_classes._base import T_CogniteResourceList, T_WritableCogniteResource, T_WriteClass
+import questionary
 from cognite.client.exceptions import CogniteAPIError, CogniteNotFoundError
 from cognite.client.utils.useful_types import SequenceNotStr
 from rich import print
@@ -24,7 +25,7 @@ from cognite_toolkit._cdf_tk.cruds import (
     ResourceCRUD,
     ResourceWorker,
 )
-from cognite_toolkit._cdf_tk.cruds._base_cruds import T_ID, Loader, T_WritableCogniteResourceList
+from cognite_toolkit._cdf_tk.cruds._base_cruds import Loader
 from cognite_toolkit._cdf_tk.data_classes import (
     BuildEnvironment,
     DeployResults,
@@ -34,9 +35,12 @@ from cognite_toolkit._cdf_tk.data_classes import (
 from cognite_toolkit._cdf_tk.data_classes._module_directories import ReadModule
 from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitCleanResourceError,
+    ToolkitMissingModuleError,
     ToolkitNotADirectoryError,
     ToolkitValidationError,
+    ToolkitValueError,
 )
+from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.tk_warnings import (
     LowSeverityWarning,
     MediumSeverityWarning,
@@ -48,6 +52,7 @@ from cognite_toolkit._cdf_tk.utils import (
     read_yaml_file,
 )
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
+from cognite_toolkit._cdf_tk.utils.useful_types import T_ID
 
 from ._utils import _print_ids_or_length
 
@@ -57,9 +62,7 @@ AVAILABLE_DATA_TYPES: tuple[str, ...] = tuple(CRUDS_BY_FOLDER_NAME)
 class CleanCommand(ToolkitCommand):
     def clean_resources(
         self,
-        loader: ResourceCRUD[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
         env_vars: EnvironmentVariables,
         read_modules: list[ReadModule],
         dry_run: bool = False,
@@ -130,7 +133,7 @@ class CleanCommand(ToolkitCommand):
             return ResourceDeployResult(name=loader.display_name)
 
     def _delete_resources(
-        self, loaded_resources: T_CogniteResourceList, loader: ResourceCRUD, dry_run: bool, verbose: bool
+        self, loaded_resources: Sequence[T_ResourceResponse], loader: ResourceCRUD, dry_run: bool, verbose: bool
     ) -> int:
         nr_of_deleted = 0
         resource_ids = loader.get_ids(loaded_resources)
@@ -155,7 +158,11 @@ class CleanCommand(ToolkitCommand):
         return nr_of_deleted
 
     def _drop_data(
-        self, loaded_resources: T_CogniteResourceList, loader: ResourceContainerCRUD, dry_run: bool, verbose: bool
+        self,
+        loaded_resources: Sequence[T_ResourceResponse],
+        loader: ResourceContainerCRUD,
+        dry_run: bool,
+        verbose: bool,
     ) -> int:
         nr_of_dropped = 0
         resource_ids = loader.get_ids(loaded_resources)
@@ -179,6 +186,23 @@ class CleanCommand(ToolkitCommand):
                 self._verbose_print_drop(resource_drop_count, resource_ids, loader, dry_run)
         return nr_of_dropped
 
+    def _interactive_module_selection(self, built_modules: list[ReadModule] | None) -> list[ReadModule] | None:
+        if not built_modules:
+            return None
+        choices = [
+            questionary.Choice(title=built_module.dir.name, value=built_module) for built_module in built_modules
+        ]
+
+        selected_modules = questionary.checkbox(
+            "Which modules would you like to clean?",
+            instruction="Use arrow up/down, press space to select item(s) and enter to save",
+            choices=choices,
+        ).ask()
+
+        if not selected_modules:
+            return None
+        return selected_modules
+
     def _verbose_print_drop(
         self, drop_count: int, resource_ids: SequenceNotStr[T_ID], loader: ResourceContainerCRUD, dry_run: bool
     ) -> None:
@@ -198,6 +222,11 @@ class CleanCommand(ToolkitCommand):
             # Count is not supported
             print(f" {prefix} all {loader.item_name} from {loader.display_name}: {_print_ids_or_length(resource_ids)}.")
 
+    def _select_modules(self, clean_state: BuildEnvironment, module_str: str | None) -> list[ReadModule] | None:
+        if module_str:
+            return [module for module in clean_state.read_modules if module.dir.name == module_str]
+        return self._interactive_module_selection(clean_state.read_modules)
+
     def execute(
         self,
         env_vars: EnvironmentVariables,
@@ -205,7 +234,9 @@ class CleanCommand(ToolkitCommand):
         build_env_name: str | None,
         dry_run: bool,
         include: list[str] | None,
+        module_str: str | None,
         verbose: bool,
+        all_modules: bool = False,
     ) -> None:
         if not build_dir.exists():
             raise ToolkitNotADirectoryError(
@@ -239,9 +270,31 @@ class CleanCommand(ToolkitCommand):
         )
 
         if not build_dir.is_dir():
-            raise ToolkitNotADirectoryError(f"'{build_dir}'. Did you forget to run `cdf-tk build` first?")
+            raise ToolkitNotADirectoryError(f"'{build_dir}'. Did you forget to run `cdf build` first?")
+
+        selected_modules: list[ReadModule]
+        if all_modules:
+            selected_modules = clean_state.read_modules or []
+            if not selected_modules:
+                raise ToolkitValueError("No modules available to clean.")
+        elif module_str:
+            selected_modules = [module for module in clean_state.read_modules if module.dir.name == module_str]
+            if not selected_modules:
+                available_module_names = {module.dir.name for module in clean_state.read_modules}
+                raise ToolkitMissingModuleError(
+                    f"No modules matched the selection: {module_str}. Available modules: {sorted(available_module_names)}"
+                )
+        else:
+            selected_modules = self._interactive_module_selection(clean_state.read_modules) or []
+            if not selected_modules:
+                raise ToolkitValueError(
+                    "No module specified with the --module option and no modules selected interactively."
+                )
 
-        selected_loaders = self.get_selected_loaders(build_dir, clean_state.read_resource_folders, include)
+        selected_resource_folders = {
+            resource_folder for module in selected_modules for resource_folder in module.resource_directories
+        }
+        selected_loaders = self.get_selected_loaders(build_dir, selected_resource_folders, include)
 
         results = DeployResults([], "clean", dry_run=dry_run)
 
@@ -268,7 +321,7 @@ class CleanCommand(ToolkitCommand):
             result = self.clean_resources(
                 loader,
                 env_vars=env_vars,
-                read_modules=clean_state.read_modules,
+                read_modules=selected_modules,
                 drop=True,
                 dry_run=dry_run,
                 drop_data=True,