cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/auth.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from __future__ import annotations
+
 
 import itertools
 import shutil
@@ -24,10 +24,13 @@ from time import sleep
 
 import questionary
 from cognite.client.data_classes.capabilities import (
+    AssetsAcl,
     Capability,
+    ExtractionConfigsAcl,
     FunctionsAcl,
     GroupsAcl,
     ProjectsAcl,
+    RelationshipsAcl,
     SessionsAcl,
 )
 from cognite.client.data_classes.iam import Group, GroupList, GroupWrite, TokenInspection
@@ -37,13 +40,14 @@ from rich.panel import Panel
 from rich.prompt import Prompt
 from rich.table import Table
 
-from cognite_toolkit._cdf_tk import loaders
+from cognite_toolkit._cdf_tk import cruds
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.constants import (
     HINT_LEAD_TEXT,
     TOOLKIT_DEMO_GROUP_NAME,
     TOOLKIT_SERVICE_PRINCIPAL_GROUP_NAME,
 )
+from cognite_toolkit._cdf_tk.cruds import ExtractionPipelineConfigCRUD
 from cognite_toolkit._cdf_tk.exceptions import (
     AuthenticationError,
     AuthorizationError,
@@ -93,16 +97,6 @@ class AuthCommand(ToolkitCommand):
             raise AuthenticationError(f"Unable to verify the credentials.\n{e}")
 
         print("[green]The credentials are valid.[/green]")
-        if no_verify:
-            return
-        print(
-            Panel(
-                "Running verification, 'cdf auth verify'...",
-                title="",
-                expand=False,
-            )
-        )
-        self.verify(client, dry_run)
 
     def _store_dotenv(self, env_vars: EnvironmentVariables) -> None:
         new_env_file = env_vars.create_dotenv_file()
@@ -236,9 +230,9 @@ class AuthCommand(ToolkitCommand):
         if not is_demo and not is_user_in_toolkit_group:
             print(
                 Panel(
-                    f"To use the Toolkit, for example, 'cdf deploy', [red]you need to switch[/red] "
-                    f"to the principal with source-id {cdf_toolkit_group.source_id!r}.",
-                    title="Switch Principal",
+                    "To use the Toolkit, for example, 'cdf deploy', [red]you need[/red] to make sure to use a service principal "
+                    f"that is a member of the group with object id {cdf_toolkit_group.source_id!r}.",
+                    title="Service Principal group membership",
                     expand=False,
                 )
             )
@@ -344,7 +338,7 @@ class AuthCommand(ToolkitCommand):
             print(f"  [bold green]OK[/] - The {existing_group.name} has all the required capabilities.")
             return []
 
-        missing_capabilities = self._merge_capabilities(missing_capabilities)
+        missing_capabilities = self.merge_capabilities(missing_capabilities)
         for s in sorted(map(str, missing_capabilities)):
             self.warn(MissingCapabilityWarning(s))
 
@@ -388,7 +382,7 @@ class AuthCommand(ToolkitCommand):
                 updated_toolkit_group.capabilities or [],
                 project=client.config.project,
             )
-        adding = self._merge_capabilities(adding)
+        adding = self.merge_capabilities(adding)
         capability_str = "capabilities" if len(adding) > 1 else "capability"
         if dry_run:
             print(f"Would have updated group {updated_toolkit_group.name} with {len(adding)} new {capability_str}.")
@@ -437,16 +431,36 @@ class AuthCommand(ToolkitCommand):
     ) -> tuple[list[Capability], dict[tuple, list[str]]]:
         loaders_by_capability_tuple: dict[tuple, list[str]] = defaultdict(list)
         capability_by_id: dict[frozenset[tuple], Capability] = {}
-        for loader_cls in loaders.RESOURCE_LOADER_LIST:
-            loader = loader_cls.create_loader(client)
-            capability = loader_cls.get_required_capability(None, read_only=False)
-            capabilities = capability if isinstance(capability, list) else [capability]
+        project_type = client.project.status().this_project.data_modeling_status
+        for crud_cls in cruds.RESOURCE_CRUD_LIST:
+            crud = crud_cls.create_loader(client)
+            if crud.prerequisite_warning() is not None:
+                continue
+            if isinstance(crud, ExtractionPipelineConfigCRUD):
+                # The Extraction Pipeline Config CRUD requires special handling.
+                # The .get_required_capability is used in the DeployCommand as well. Since, there is no way to no
+                # the extraction pipeline ID or dataSetId from an ExtractionPipelineConfigWrite object, we do not
+                # check those there. If we returned the full capability, it would always have to be all scoped.
+                # That is too restrictive in the deploy command, so we return an empty list, essentially not checking
+                # anything there. Here, we want to add the all scoped capability, so that the Toolkit group gets the
+                # correct capability.
+                capabilities: list[Capability] = [
+                    ExtractionConfigsAcl(
+                        [ExtractionConfigsAcl.Action.Read, ExtractionConfigsAcl.Action.Write],
+                        ExtractionConfigsAcl.Scope.All(),
+                    )
+                ]
+            else:
+                capability = crud_cls.get_required_capability(None, read_only=False)
+                capabilities = capability if isinstance(capability, list) else [capability]
             for cap in capabilities:
+                if project_type == "DATA_MODELING_ONLY" and isinstance(cap, AssetsAcl | RelationshipsAcl):
+                    continue
                 id_ = frozenset(cap.as_tuples())
                 if id_ not in capability_by_id:
                     capability_by_id[id_] = cap
                 for cap_tuple in cap.as_tuples():
-                    loaders_by_capability_tuple[cap_tuple].append(loader.display_name)
+                    loaders_by_capability_tuple[cap_tuple].append(crud.display_name)
         return list(capability_by_id.values()), loaders_by_capability_tuple
 
     def check_has_any_access(self, client: ToolkitClient) -> TokenInspection:
@@ -602,15 +616,21 @@ class AuthCommand(ToolkitCommand):
         return extra
 
     @staticmethod
-    def _merge_capabilities(capability_list: list[Capability]) -> list[Capability]:
+    def merge_capabilities(capability_list: list[Capability]) -> list[Capability]:
         """Merges capabilities that have the same ACL and Scope"""
-        actions_by_scope_and_cls: dict[tuple[type[Capability], Capability.Scope], set[Capability.Action]] = defaultdict(
+        actions_by_scope_and_cls: dict[tuple[type[Capability], frozenset[tuple]], set[Capability.Action]] = defaultdict(
             set
         )
         for capability in capability_list:
-            actions_by_scope_and_cls[(type(capability), capability.scope)].update(capability.actions)
+            scope_key = frozenset(capability.scope.as_tuples())
+            actions_by_scope_and_cls[(type(capability), scope_key)].update(capability.actions)
+
+        scope_map: dict[tuple[type[Capability], frozenset[tuple]], Capability.Scope] = {
+            (type(cap), frozenset(cap.scope.as_tuples())): cap.scope for cap in capability_list
+        }
+
         return [
-            cap_cls(actions=list(actions), scope=scope, allow_unknown=False)
+            cap_cls(actions=list(actions), scope=scope_map[(cap_cls, scope)], allow_unknown=False)
             for (cap_cls, scope), actions in actions_by_scope_and_cls.items()
         ]
 

cognite_toolkit/_cdf_tk/commands/build_cmd.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import contextlib
 import re
 from collections import defaultdict
@@ -22,13 +20,28 @@ from cognite_toolkit._cdf_tk.constants import (
     _RUNNING_IN_BROWSER,
     BUILD_FOLDER_ENCODING,
     DEFAULT_ENV,
-    DEV_ONLY_MODULES,
     HINT_LEAD_TEXT,
     ROOT_MODULES,
     TEMPLATE_VARS_FILE_SUFFIXES,
     URL,
     YAML_SUFFIX,
 )
+from cognite_toolkit._cdf_tk.cruds import (
+    ContainerCRUD,
+    DataCRUD,
+    DataModelCRUD,
+    DataSetsCRUD,
+    ExtractionPipelineConfigCRUD,
+    FileCRUD,
+    LocationFilterCRUD,
+    NodeCRUD,
+    RawDatabaseCRUD,
+    RawTableCRUD,
+    ResourceCRUD,
+    SpaceCRUD,
+    TransformationCRUD,
+    ViewCRUD,
+)
 from cognite_toolkit._cdf_tk.data_classes import (
     BuildConfigYAML,
     BuildDestinationFile,
@@ -45,29 +58,10 @@ from cognite_toolkit._cdf_tk.data_classes import (
     SourceLocationLazy,
 )
 from cognite_toolkit._cdf_tk.exceptions import (
-    ToolkitDuplicatedModuleError,
-    ToolkitEnvError,
     ToolkitError,
-    ToolkitMissingModuleError,
     ToolkitYAMLFormatError,
 )
 from cognite_toolkit._cdf_tk.hints import Hint, ModuleDefinition, verify_module_directory
-from cognite_toolkit._cdf_tk.loaders import (
-    ContainerLoader,
-    DataLoader,
-    DataModelLoader,
-    DataSetsLoader,
-    ExtractionPipelineConfigLoader,
-    FileLoader,
-    LocationFilterLoader,
-    NodeLoader,
-    RawDatabaseLoader,
-    RawTableLoader,
-    ResourceLoader,
-    SpaceLoader,
-    TransformationLoader,
-    ViewLoader,
-)
 from cognite_toolkit._cdf_tk.tk_warnings import (
     DuplicatedItemWarning,
     FileReadWarning,
@@ -79,7 +73,7 @@ from cognite_toolkit._cdf_tk.tk_warnings import (
 )
 from cognite_toolkit._cdf_tk.tk_warnings.fileread import MissingRequiredIdentifierWarning
 from cognite_toolkit._cdf_tk.utils import (
-    calculate_str_or_file_hash,
+    calculate_hash,
     humanize_collection,
     quote_int_value_by_key_in_yaml,
     read_yaml_content,
@@ -91,8 +85,8 @@ from cognite_toolkit._cdf_tk.utils.file import safe_rmtree
 from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules
 from cognite_toolkit._cdf_tk.validation import (
     validate_data_set_is_set,
+    validate_module_selection,
     validate_modules_variables,
-    validate_resource_yaml,
     validate_resource_yaml_pydantic,
 )
 from cognite_toolkit._version import __version__
@@ -106,14 +100,14 @@ class BuildCommand(ToolkitCommand):
         silent: bool = False,
     ) -> None:
         super().__init__(print_warning, skip_tracking, silent)
-        self.existing_resources_by_loader: dict[type[ResourceLoader], set[Hashable]] = defaultdict(set)
-        self.instantiated_loaders: dict[type[ResourceLoader], ResourceLoader] = {}
+        self.existing_resources_by_loader: dict[type[ResourceCRUD], set[Hashable]] = defaultdict(set)
+        self.instantiated_loaders: dict[type[ResourceCRUD], ResourceCRUD] = {}
 
         # Built State
         self._module_names_by_variable_key: dict[str, list[str]] = defaultdict(list)
         self._builder_by_resource_folder: dict[str, Builder] = {}
-        self._ids_by_resource_type: dict[type[ResourceLoader], dict[Hashable, SourceLocation]] = defaultdict(dict)
-        self._dependencies_by_required: dict[tuple[type[ResourceLoader], Hashable], list[tuple[Hashable, Path]]] = (
+        self._ids_by_resource_type: dict[type[ResourceCRUD], dict[Hashable, SourceLocation]] = defaultdict(dict)
+        self._dependencies_by_required: dict[tuple[type[ResourceCRUD], Hashable], list[tuple[Hashable, Path]]] = (
             defaultdict(list)
         )
         self._has_built = False
@@ -145,6 +139,11 @@ class BuildCommand(ToolkitCommand):
         if selected:
             config.environment.selected = parse_user_selected_modules(selected, organization_dir)
 
+        # tracking which project the module is being built for to trace promotion
+        if client:
+            self._additional_tracking_info.project = client.config.project
+            self._additional_tracking_info.cluster = client.config.cdf_cluster
+
         directory_name = "current directory" if organization_dir == Path(".") else f"project '{organization_dir!s}'"
         root_modules = [
             module_dir for root_module in ROOT_MODULES if (module_dir := organization_dir / root_module).exists()
@@ -205,7 +204,11 @@ class BuildCommand(ToolkitCommand):
 
         user_selected_modules = config.environment.get_selected_modules(packages)
         modules = ModuleDirectories.load(organization_dir, user_selected_modules)
-        self._validate_modules(modules, config, packages, user_selected_modules, organization_dir)
+        module_warnings = validate_module_selection(modules, config, packages, user_selected_modules, organization_dir)
+        if module_warnings:
+            self.warning_list.extend(module_warnings)
+            if self.print_warning:
+                print(str(module_warnings))
 
         if verbose:
             self.console("Selected packages:")
@@ -325,9 +328,17 @@ class BuildCommand(ToolkitCommand):
                     warning_count=module_warnings,
                     status=built_status,
                     iteration=iteration,
+                    module_id=module.module_id,
+                    package_id=module.package_id,
                 )
                 build.append(built_module)
-                self.tracker.track_module_build(built_module)
+
+                if module.package_id:
+                    self._additional_tracking_info.package_ids.add(module.package_id)
+
+                if module.module_id:
+                    self._additional_tracking_info.module_ids.add(module.module_id)
+
         return build
 
     def _build_module_resources(
@@ -348,6 +359,15 @@ class BuildCommand(ToolkitCommand):
         for resource_name, resource_files in module.source_paths_by_resource_folder.items():
             source_files = self._replace_variables(resource_files, module_variables, resource_name, module.dir, verbose)
 
+            if resource_name == "data_models":
+                resource_name = "data_modeling"
+                self.warn(
+                    MediumSeverityWarning(
+                        "The resource folder 'data_models' is deprecated and will be removed in v1.0. "
+                        "Please rename the folder to 'data_modeling'."
+                    )
+                )
+
             builder = self._get_builder(build_dir, resource_name)
 
             built_resources = BuiltResourceList[Hashable]()
@@ -356,12 +376,12 @@ class BuildCommand(ToolkitCommand):
                     for warning in destination:
                         self.warn(warning)
                     continue
-                if destination.loader is FileLoader:
+                if destination.loader is FileCRUD:
                     # This is a content file that we should not copy to the build directory.
                     continue
 
                 safe_write(destination.path, destination.content, encoding=BUILD_FOLDER_ENCODING)
-                if issubclass(destination.loader, DataLoader):
+                if issubclass(destination.loader, DataCRUD):
                     continue
 
                 file_warnings, identifiers_kind_pairs = self.check_built_resource(
@@ -399,66 +419,10 @@ class BuildCommand(ToolkitCommand):
 
     def _get_builder(self, build_dir: Path, resource_name: str) -> Builder:
         if resource_name not in self._builder_by_resource_folder:
-            self._builder_by_resource_folder[resource_name] = create_builder(resource_name, build_dir)
+            self._builder_by_resource_folder[resource_name] = create_builder(resource_name, build_dir, self.warn)
         builder = self._builder_by_resource_folder[resource_name]
         return builder
 
-    def _validate_modules(
-        self,
-        modules: ModuleDirectories,
-        config: BuildConfigYAML,
-        packages: dict[str, list[str]],
-        selected_modules: set[str | Path],
-        organization_dir: Path,
-    ) -> None:
-        # Validations: Ambiguous selection.
-        selected_names = {s for s in config.environment.selected if isinstance(s, str)}
-        if duplicate_modules := {
-            module_name: paths
-            for module_name, paths in modules.as_path_by_name().items()
-            if len(paths) > 1 and module_name in selected_names
-        }:
-            # If the user has selected a module by name, and there are multiple modules with that name, raise an error.
-            # Note, if the user uses a path to select a module, this error will not be raised.
-            raise ToolkitDuplicatedModuleError(
-                f"Ambiguous module selected in config.{config.environment.name}.yaml:", duplicate_modules
-            )
-        # Package Referenced Modules Exists
-        for package, package_modules in packages.items():
-            if package not in selected_names:
-                # We do not check packages that are not selected.
-                # Typically, the user will delete the modules that are irrelevant for them;
-                # thus we only check the selected packages.
-                continue
-            if missing_packages := set(package_modules) - modules.available_names:
-                ToolkitMissingModuleError(
-                    f"Package {package} defined in {CDFToml.file_name!s} is referring "
-                    f"the following missing modules {missing_packages}."
-                )
-
-        # Selected modules does not exists
-        if missing_modules := set(selected_modules) - modules.available:
-            hint = ModuleDefinition.long(missing_modules, organization_dir)
-            raise ToolkitMissingModuleError(
-                f"The following selected modules are missing, please check path: {missing_modules}.\n{hint}"
-            )
-
-        # Nothing is Selected
-        if not modules.selected:
-            raise ToolkitEnvError(
-                f"No selected modules specified in {config.filepath!s}, have you configured "
-                f"the environment ({config.environment.name})?"
-            )
-
-        dev_modules = modules.available_names & DEV_ONLY_MODULES
-        if dev_modules and config.environment.validation_type != "dev":
-            self.warn(
-                MediumSeverityWarning(
-                    "The following modules should [bold]only[/bold] be used a in CDF Projects designated as dev (development): "
-                    f"{humanize_collection(dev_modules)!r}",
-                )
-            )
-
     def _replace_variables(
         self,
         resource_files: Sequence[Path],
@@ -480,9 +444,9 @@ class BuildCommand(ToolkitCommand):
             # We cannot use the content as the basis for hash as this have been encoded.
             # Instead, we use the source path, which will hash the bytes of the file directly,
             # which is what we do in the deploy step to verify that the source file has not changed.
-            source = SourceLocationEager(source_path, calculate_str_or_file_hash(source_path, shorten=True))
+            source = SourceLocationEager(source_path, calculate_hash(source_path, shorten=True))
 
-            content = variables.replace(content, source_path.suffix)
+            content = variables.replace(content, source_path)
 
             replace_warnings = self._check_variables_replaced(content, module_dir, source_path)
 
@@ -491,15 +455,15 @@ class BuildCommand(ToolkitCommand):
                 continue
 
             if resource_name in {
-                TransformationLoader.folder_name,
-                DataModelLoader.folder_name,
-                LocationFilterLoader.folder_name,
+                TransformationCRUD.folder_name,
+                DataModelCRUD.folder_name,
+                LocationFilterCRUD.folder_name,
             }:
                 # Ensure that all keys that are version gets read as strings.
                 # This is required by DataModels, Views, and Transformations that reference DataModels and Views.
                 content = quote_int_value_by_key_in_yaml(content, key="version")
 
-            if resource_name in ExtractionPipelineConfigLoader.folder_name:
+            if resource_name in ExtractionPipelineConfigCRUD.folder_name:
                 # Ensure that the config variables are stings.
                 # This is required by ExtractionPipelineConfig
                 content = stringify_value_by_key_in_yaml(content, key="config")
@@ -561,7 +525,7 @@ class BuildCommand(ToolkitCommand):
         for loader_cls, id_ in missing_dependencies:
             if self._is_system_resource(loader_cls, id_):
                 continue
-            elif loader_cls is DataSetsLoader and id_ == "":
+            elif loader_cls is DataSetsCRUD and id_ == "":
                 # Special case used by the location filter to indicate filter out all classical resources.
                 continue
             elif client and self._check_resource_exists_in_cdf(client, loader_cls, id_):
@@ -573,10 +537,11 @@ class BuildCommand(ToolkitCommand):
                 (required, path.relative_to(project_config_dir))
                 for required, path in self._dependencies_by_required[(loader_cls, id_)]
             }
-            self.warn(MissingDependencyWarning(loader_cls.resource_cls.__name__, id_, required_by))
+            has_checked_cdf = client is not None
+            self.warn(MissingDependencyWarning(loader_cls.resource_cls.__name__, id_, required_by, has_checked_cdf))
 
     def _check_resource_exists_in_cdf(
-        self, client: ToolkitClient, loader_cls: type[ResourceLoader], id_: Hashable
+        self, client: ToolkitClient, loader_cls: type[ResourceCRUD], id_: Hashable
     ) -> bool:
         """Check is the resource exists in the CDF project. If there are any issues assume it does not exist."""
         if id_ in self.existing_resources_by_loader[loader_cls]:
@@ -589,19 +554,18 @@ class BuildCommand(ToolkitCommand):
             retrieved = loader.retrieve([id_])
             if retrieved:
                 self.existing_resources_by_loader[loader_cls].add(id_)
-            return True
+                return True
         return False
 
     def check_built_resource(
         self,
         parsed: dict[str, Any] | list[dict[str, Any]],
-        loader: type[ResourceLoader],
+        loader: type[ResourceCRUD],
         source: SourceLocation,
     ) -> tuple[WarningList[FileReadWarning], list[tuple[Hashable, str]]]:
         warning_list = WarningList[FileReadWarning]()
 
-        if loader.yaml_cls:
-            warning_list.extend(validate_resource_yaml_pydantic(parsed, loader.yaml_cls, source.path))
+        warning_list.extend(validate_resource_yaml_pydantic(parsed, loader.yaml_cls, source.path))
 
         is_dict_item = isinstance(parsed, dict)
         items = [parsed] if isinstance(parsed, dict) else parsed
@@ -616,10 +580,10 @@ class BuildCommand(ToolkitCommand):
             try:
                 identifier = item_loader.get_id(item)
             except KeyError as error:
-                if loader is RawTableLoader:
+                if loader is RawTableCRUD:
                     try:
-                        identifier = RawDatabaseLoader.get_id(item)
-                        item_loader = RawDatabaseLoader
+                        identifier = RawDatabaseCRUD.get_id(item)
+                        item_loader = RawDatabaseCRUD
                     except KeyError:
                         warning_list.append(
                             MissingRequiredIdentifierWarning(source.path, element_no, tuple(), error.args)
@@ -654,12 +618,6 @@ class BuildCommand(ToolkitCommand):
                 for dependency in dependencies:
                     self._dependencies_by_required[dependency].append((identifier, source.path))
 
-            api_spec = item_loader.safe_get_write_cls_parameter_spec()
-            if api_spec is not None and item_loader.yaml_cls is None:
-                # If we have a yaml cls this is already validated on the file as a whole.
-                resource_warnings = validate_resource_yaml(parsed, api_spec, source.path, element_no)
-                warning_list.extend(resource_warnings)
-
             data_set_warnings = validate_data_set_is_set(items, loader.resource_cls, source.path)
             warning_list.extend(data_set_warnings)
 
@@ -669,12 +627,12 @@ class BuildCommand(ToolkitCommand):
         return warning_list, identifier_kind_pairs
 
     @staticmethod
-    def _is_system_resource(resource_cls: type[ResourceLoader], id_: Hashable) -> bool:
+    def _is_system_resource(resource_cls: type[ResourceCRUD], id_: Hashable) -> bool:
         """System resources are deployed to all CDF project and should not be checked for dependencies."""
-        if resource_cls is SpaceLoader and isinstance(id_, str) and id_.startswith("cdf_"):
+        if resource_cls is SpaceCRUD and isinstance(id_, str) and id_.startswith("cdf_"):
             return True
         elif (
-            resource_cls in {ContainerLoader, ViewLoader, DataModelLoader, NodeLoader}
+            resource_cls in {ContainerCRUD, ViewCRUD, DataModelCRUD, NodeCRUD}
             and hasattr(id_, "space")
             and id_.space.startswith("cdf_")
         ):