cognite-toolkit 0.6.39__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_cdf_tk/validation.py

@@ -4,25 +4,35 @@ from pathlib import Path
 from typing import Any, TypeVar
 
 from cognite.client.data_classes._base import CogniteObject
-from cognite.client.utils._text import to_camel_case, to_snake_case
+from cognite.client.utils._text import to_snake_case
 from pydantic import BaseModel, TypeAdapter, ValidationError
 from pydantic_core import ErrorDetails
 
-from cognite_toolkit._cdf_tk._parameters import ParameterSpecSet, read_parameters_from_dict
-from cognite_toolkit._cdf_tk.cruds import NodeCRUD
-from cognite_toolkit._cdf_tk.data_classes import BuildVariables
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
+from cognite_toolkit._cdf_tk.constants import DEV_ONLY_MODULES
+from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML, BuildVariables, ModuleDirectories
+from cognite_toolkit._cdf_tk.exceptions import (
+    ToolkitDuplicatedModuleError,
+    ToolkitEnvError,
+    ToolkitMissingModuleError,
+)
+from cognite_toolkit._cdf_tk.hints import ModuleDefinition
 from cognite_toolkit._cdf_tk.resource_classes import BaseModelResource
 from cognite_toolkit._cdf_tk.tk_warnings import (
-    CaseTypoWarning,
     DataSetMissingWarning,
-    MissingRequiredParameterWarning,
+    MediumSeverityWarning,
     TemplateVariableWarning,
-    UnusedParameterWarning,
     WarningList,
 )
 from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+from cognite_toolkit._cdf_tk.utils import humanize_collection
 
-__all__ = ["validate_data_set_is_set", "validate_modules_variables", "validate_resource_yaml"]
+__all__ = [
+    "humanize_validation_error",
+    "validate_data_set_is_set",
+    "validate_module_selection",
+    "validate_modules_variables",
+]
 
 
 T_BaseModel = TypeVar("T_BaseModel", bound=BaseModel)
@@ -72,55 +82,6 @@ def validate_data_set_is_set(
     return warning_list
 
 
-def validate_resource_yaml(
-    data: dict | list, spec: ParameterSpecSet, source_file: Path, element: int | None = None
-) -> WarningList:
-    if spec.spec_name == NodeCRUD.__name__:
-        # Special case for NodeLoader as it has options for API call parameters
-        if isinstance(data, list):
-            return _validate_resource_yaml(data, spec, source_file)
-        elif isinstance(data, dict) and "node" in data:
-            return _validate_resource_yaml(data["node"], spec, source_file)
-        elif isinstance(data, dict) and "nodes" in data:
-            return _validate_resource_yaml(data["nodes"], spec, source_file)
-        else:
-            return _validate_resource_yaml(data, spec, source_file)
-    else:
-        return _validate_resource_yaml(data, spec, source_file, element)
-
-
-def _validate_resource_yaml(
-    data: dict | list, spec: ParameterSpecSet, source_file: Path, element: int | None = None
-) -> WarningList:
-    warnings: WarningList = WarningList()
-    if isinstance(data, list):
-        for no, item in enumerate(data, 1):
-            warnings.extend(_validate_resource_yaml(item, spec, source_file, no))
-        return warnings
-    elif not isinstance(data, dict):
-        raise NotImplementedError("Note: This function only supports top-level and lists dictionaries.")
-
-    actual_parameters = read_parameters_from_dict(data)
-    unused_parameters = actual_parameters - spec
-    unused_cased = unused_parameters.as_camel_case() - spec
-    typo_parameters = unused_parameters - unused_cased
-    for parameter in typo_parameters:
-        key = parameter.key
-        warnings.append(CaseTypoWarning(source_file, element, parameter.path, key, to_camel_case(key)))
-
-    unused_parameters = unused_parameters - typo_parameters
-    for parameter in unused_parameters:
-        key = parameter.key
-        warnings.append(UnusedParameterWarning(source_file, element, parameter.path, key))
-
-    # Only checking the top level for now. This can be expanded to check nested parameters.
-    missing = spec.required(level=1) - actual_parameters
-    for spec_param in missing:
-        warnings.append(MissingRequiredParameterWarning(source_file, element, spec_param.path, spec_param.key))
-
-    return warnings
-
-
 def validate_resource_yaml_pydantic(
     data: dict[str, object] | list[dict[str, object]], validation_cls: type[BaseModelResource], source_file: Path
 ) -> WarningList:
@@ -136,18 +97,17 @@ def validate_resource_yaml_pydantic(
 
     """
     warning_list: WarningList = WarningList()
-    if isinstance(data, dict):
-        try:
+    try:
+        if isinstance(data, dict):
             validation_cls.model_validate(data, strict=True)
-        except ValidationError as e:
-            warning_list.append(ResourceFormatWarning(source_file, tuple(_humanize_validation_error(e))))
-    elif isinstance(data, list):
-        try:
+        elif isinstance(data, list):
             TypeAdapter(list[validation_cls]).validate_python(data)  # type: ignore[valid-type]
-        except ValidationError as e:
-            warning_list.append(ResourceFormatWarning(source_file, tuple(_humanize_validation_error(e))))
-    else:
-        raise ValueError(f"Expected a dictionary or list of dictionaries, got {type(data)}.")
+        else:
+            raise ValueError(f"Expected a dictionary or list of dictionaries, got {type(data)}.")
+    except ValidationError as e:
+        printable_errors = tuple(humanize_validation_error(e))
+        if printable_errors:
+            warning_list.append(ResourceFormatWarning(source_file, printable_errors))
     return warning_list
 
 
@@ -168,10 +128,10 @@ def instantiate_class(
     try:
         return validation_cls.model_validate(data, strict=strict)
     except ValidationError as e:
-        return ResourceFormatWarning(source_file, tuple(_humanize_validation_error(e)))
+        return ResourceFormatWarning(source_file, tuple(humanize_validation_error(e)))
 
 
-def _humanize_validation_error(error: ValidationError) -> list[str]:
+def humanize_validation_error(error: ValidationError) -> list[str]:
     """Converts a ValidationError to a human-readable format.
 
     This overwrites the default error messages from Pydantic to be better suited for Toolkit users.
@@ -261,3 +221,70 @@ def as_json_path(loc: tuple[str | int, ...]) -> str:
 
     suffix = ".".join([str(x) if isinstance(x, str) else f"[{x + 1}]" for x in loc]).replace(".[", "[")
     return f"{prefix}{suffix}"
+
+
+def validate_module_selection(
+    modules: ModuleDirectories,
+    config: BuildConfigYAML,
+    packages: dict[str, list[str]],
+    selected_modules: set[str | Path],
+    organization_dir: Path,
+) -> WarningList:
+    """Validates module selection and returns warnings for non-critical issues.
+
+    Critical errors (duplicate modules, missing modules, no modules selected) are still raised
+    as exceptions as they prevent the build from proceeding.
+    """
+    warnings: WarningList = WarningList()
+
+    # Validations: Ambiguous selection.
+    selected_names = {s for s in config.environment.selected if isinstance(s, str)}
+    if duplicate_modules := {
+        module_name: paths
+        for module_name, paths in modules.as_path_by_name().items()
+        if len(paths) > 1 and module_name in selected_names
+    }:
+        # If the user has selected a module by name, and there are multiple modules with that name, raise an error.
+        # Note, if the user uses a path to select a module, this error will not be raised.
+        raise ToolkitDuplicatedModuleError(
+            f"Ambiguous module selected in config.{config.environment.name}.yaml:", duplicate_modules
+        )
+
+    # Package Referenced Modules Exists
+    for package, package_modules in packages.items():
+        if package not in selected_names:
+            # We do not check packages that are not selected.
+            # Typically, the user will delete the modules that are irrelevant for them;
+            # thus we only check the selected packages.
+            continue
+        if missing_packages := set(package_modules) - modules.available_names:
+            raise ToolkitMissingModuleError(
+                f"Package {package} defined in {CDFToml.file_name!s} is referring "
+                f"the following missing modules {missing_packages}."
+            )
+
+    # Selected modules does not exists
+    if missing_modules := set(selected_modules) - modules.available:
+        hint = ModuleDefinition.long(missing_modules, organization_dir)
+        raise ToolkitMissingModuleError(
+            f"The following selected modules are missing, please check path: {missing_modules}.\n{hint}"
+        )
+
+    # Nothing is Selected
+    if not modules.selected:
+        raise ToolkitEnvError(
+            f"No selected modules specified in {config.filepath!s}, have you configured "
+            f"the environment ({config.environment.name})?"
+        )
+
+    # Dev modules warning (non-critical)
+    dev_modules = modules.available_names & DEV_ONLY_MODULES
+    if dev_modules and config.environment.validation_type != "dev":
+        warnings.append(
+            MediumSeverityWarning(
+                "The following modules should [bold]only[/bold] be used a in CDF Projects designated as dev (development): "
+                f"{humanize_collection(dev_modules)!r}",
+            )
+        )
+
+    return warnings

cognite_toolkit/_repo_files/.gitignore

@@ -174,6 +174,7 @@ celerybeat.pid
 
 # Environments
 .env
+*.env
 .venv
 .local/
 env/
@@ -276,3 +277,4 @@ $RECYCLE.BIN/
 build.*
 build/
 function_local_venvs/
+data/

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.39
+      image: cognite/toolkit:0.7.30
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.39
+      image: cognite/toolkit:0.7.30
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -0,0 +1,18 @@
+[cdf]#<PLACEHOLDER>
+default_env = "<DEFAULT_ENV_PLACEHOLDER>"
+
+[modules]
+# This is the version of the modules. It should not be changed manually.
+# It will be updated by the 'cdf modules upgrade' command.
+version = "0.7.30"
+
+
+[plugins]
+dump = false
+dev = false
+data = false
+
+
+[library.toolkit-data]
+url = "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip" 
+checksum = "sha256:b70c207a78d6c94e0977a733571420b250ca641ac8802d8310f1ffe05bf7b2eb"

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.6.39"
+__version__ = "0.7.30"

cognite_toolkit/config.dev.yaml

@@ -0,0 +1,13 @@
+environment:
+  name: dev
+  project: infield-experimental-test-aws
+  validation-type: dev
+  selected:
+    - modules/eriks-location
+
+variables:
+  modules:
+    eriks-location:
+      - location: erik-infield-cdm2
+        location_name: Oslo CDM - Erik tookit
+        location_description: Used to test toolkit ingestion for InField cdm

{cognite_toolkit-0.6.39.dist-info → cognite_toolkit-0.7.30.dist-info}/METADATA

@@ -1,38 +1,38 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.39
+Version: 0.7.30
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
-Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
-Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases
-Project-URL: GitHub, https://github.com/cognitedata/toolkit
-Project-URL: Documentation, https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library
+Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>
 License-Expression: Apache-2.0
-License-File: LICENSE
-Requires-Python: >=3.10
-Requires-Dist: cognite-sdk<8.0.0,>=7.83.0
-Requires-Dist: filelock>=3.18.0
-Requires-Dist: httpx>=0.28.1
-Requires-Dist: mixpanel>=4.10.1
-Requires-Dist: packaging>=25
-Requires-Dist: pandas<3.0.0,>=1.5.3
-Requires-Dist: pip>=25.0.1
-Requires-Dist: pydantic>=2.11.0
-Requires-Dist: python-dateutil>=2.9.0
 Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: cognite-sdk>=7.87.0,<8.0.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: pandas>=1.5.3,<3.0.0
 Requires-Dist: pyyaml>=6.0.1
-Requires-Dist: questionary>=2.0.1
+Requires-Dist: typer>=0.12.0,<1.0.0
 Requires-Dist: rich>=13.9.4
-Requires-Dist: sentry-sdk>=2.1.0
-Requires-Dist: toml>=0.10.2
-Requires-Dist: tomli<3.0.0,>=2.0.1; python_version < '3.11'
-Requires-Dist: typer<1.0.0,>=0.12.0
+Requires-Dist: questionary>=2.0.1
+Requires-Dist: tomli>=2.0.1,<3.0.0 ; python_full_version < '3.11'
+Requires-Dist: packaging>=25
 Requires-Dist: typing-extensions>=4.0.0
+Requires-Dist: toml>=0.10.2
+Requires-Dist: sentry-sdk>=2.1.0
+Requires-Dist: mixpanel>=4.10.1
+Requires-Dist: pydantic>=2.11.0
+Requires-Dist: python-dateutil>=2.9.0
+Requires-Dist: pip>=25.0.1
+Requires-Dist: filelock>=3.18.0
+Requires-Dist: sqlparse>=0.5.3 ; extra == 'sql'
+Requires-Dist: pyarrow>=20.0.0 ; extra == 'table'
+Requires-Dist: openpyxl>=3.1.5 ; extra == 'table'
+Requires-Python: >=3.10
+Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases
+Project-URL: Documentation, https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library
+Project-URL: GitHub, https://github.com/cognitedata/toolkit
+Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Provides-Extra: sql
-Requires-Dist: sqlparse>=0.5.3; extra == 'sql'
 Provides-Extra: table
-Requires-Dist: openpyxl>=3.1.5; extra == 'table'
-Requires-Dist: pyarrow>=20.0.0; extra == 'table'
 Description-Content-Type: text/markdown
 
 # Cognite Data Fusion Toolkit