cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/validation.py

@@ -1,31 +1,41 @@
-from __future__ import annotations
-
 import inspect
 import re
 from pathlib import Path
-from typing import Any
+from typing import Any, TypeVar
 
 from cognite.client.data_classes._base import CogniteObject
-from cognite.client.utils._text import to_camel_case, to_snake_case
-from pydantic import TypeAdapter, ValidationError
+from cognite.client.utils._text import to_snake_case
+from pydantic import BaseModel, TypeAdapter, ValidationError
 from pydantic_core import ErrorDetails
 
-from cognite_toolkit._cdf_tk._parameters import ParameterSpecSet, read_parameters_from_dict
-from cognite_toolkit._cdf_tk.data_classes import BuildVariables
-from cognite_toolkit._cdf_tk.loaders import NodeLoader
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
+from cognite_toolkit._cdf_tk.constants import DEV_ONLY_MODULES
+from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML, BuildVariables, ModuleDirectories
+from cognite_toolkit._cdf_tk.exceptions import (
+    ToolkitDuplicatedModuleError,
+    ToolkitEnvError,
+    ToolkitMissingModuleError,
+)
+from cognite_toolkit._cdf_tk.hints import ModuleDefinition
 from cognite_toolkit._cdf_tk.resource_classes import BaseModelResource
 from cognite_toolkit._cdf_tk.tk_warnings import (
-    CaseTypoWarning,
     DataSetMissingWarning,
-    MissingRequiredParameterWarning,
+    MediumSeverityWarning,
     TemplateVariableWarning,
-    UnusedParameterWarning,
     WarningList,
 )
+from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+from cognite_toolkit._cdf_tk.utils import humanize_collection
 
-__all__ = ["validate_data_set_is_set", "validate_modules_variables", "validate_resource_yaml"]
+__all__ = [
+    "humanize_validation_error",
+    "validate_data_set_is_set",
+    "validate_module_selection",
+    "validate_modules_variables",
+]
 
-from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+
+T_BaseModel = TypeVar("T_BaseModel", bound=BaseModel)
 
 
 def validate_modules_variables(variables: BuildVariables, filepath: Path) -> WarningList:
@@ -72,55 +82,6 @@ def validate_data_set_is_set(
     return warning_list
 
 
-def validate_resource_yaml(
-    data: dict | list, spec: ParameterSpecSet, source_file: Path, element: int | None = None
-) -> WarningList:
-    if spec.spec_name == NodeLoader.__name__:
-        # Special case for NodeLoader as it has options for API call parameters
-        if isinstance(data, list):
-            return _validate_resource_yaml(data, spec, source_file)
-        elif isinstance(data, dict) and "node" in data:
-            return _validate_resource_yaml(data["node"], spec, source_file)
-        elif isinstance(data, dict) and "nodes" in data:
-            return _validate_resource_yaml(data["nodes"], spec, source_file)
-        else:
-            return _validate_resource_yaml(data, spec, source_file)
-    else:
-        return _validate_resource_yaml(data, spec, source_file, element)
-
-
-def _validate_resource_yaml(
-    data: dict | list, spec: ParameterSpecSet, source_file: Path, element: int | None = None
-) -> WarningList:
-    warnings: WarningList = WarningList()
-    if isinstance(data, list):
-        for no, item in enumerate(data, 1):
-            warnings.extend(_validate_resource_yaml(item, spec, source_file, no))
-        return warnings
-    elif not isinstance(data, dict):
-        raise NotImplementedError("Note: This function only supports top-level and lists dictionaries.")
-
-    actual_parameters = read_parameters_from_dict(data)
-    unused_parameters = actual_parameters - spec
-    unused_cased = unused_parameters.as_camel_case() - spec
-    typo_parameters = unused_parameters - unused_cased
-    for parameter in typo_parameters:
-        key = parameter.key
-        warnings.append(CaseTypoWarning(source_file, element, parameter.path, key, to_camel_case(key)))
-
-    unused_parameters = unused_parameters - typo_parameters
-    for parameter in unused_parameters:
-        key = parameter.key
-        warnings.append(UnusedParameterWarning(source_file, element, parameter.path, key))
-
-    # Only checking the top level for now. This can be expanded to check nested parameters.
-    missing = spec.required(level=1) - actual_parameters
-    for spec_param in missing:
-        warnings.append(MissingRequiredParameterWarning(source_file, element, spec_param.path, spec_param.key))
-
-    return warnings
-
-
 def validate_resource_yaml_pydantic(
     data: dict[str, object] | list[dict[str, object]], validation_cls: type[BaseModelResource], source_file: Path
 ) -> WarningList:
@@ -136,22 +97,41 @@ def validate_resource_yaml_pydantic(
 
     """
     warning_list: WarningList = WarningList()
-    if isinstance(data, dict):
-        try:
+    try:
+        if isinstance(data, dict):
             validation_cls.model_validate(data, strict=True)
-        except ValidationError as e:
-            warning_list.append(ResourceFormatWarning(source_file, tuple(_humanize_validation_error(e))))
-    elif isinstance(data, list):
-        try:
+        elif isinstance(data, list):
             TypeAdapter(list[validation_cls]).validate_python(data)  # type: ignore[valid-type]
-        except ValidationError as e:
-            warning_list.append(ResourceFormatWarning(source_file, tuple(_humanize_validation_error(e))))
-    else:
-        raise ValueError(f"Expected a dictionary or list of dictionaries, got {type(data)}.")
+        else:
+            raise ValueError(f"Expected a dictionary or list of dictionaries, got {type(data)}.")
+    except ValidationError as e:
+        printable_errors = tuple(humanize_validation_error(e))
+        if printable_errors:
+            warning_list.append(ResourceFormatWarning(source_file, printable_errors))
     return warning_list
 
 
-def _humanize_validation_error(error: ValidationError) -> list[str]:
+def instantiate_class(
+    data: dict[str, Any], validation_cls: type[T_BaseModel], source_file: Path, strict: bool = False
+) -> T_BaseModel | ResourceFormatWarning:
+    """Instantiates a class from a dictionary using the given pydantic model.
+
+    Args:
+        data: The data to instantiate the class from.
+        validation_cls: The pydantic model to use for instantiation.
+        source_file: The source file of the resource.
+        strict: Whether to enforce types strictly.
+
+    Returns:
+        The instantiated class or a ResourceFormatWarning if validation failed.
+    """
+    try:
+        return validation_cls.model_validate(data, strict=strict)
+    except ValidationError as e:
+        return ResourceFormatWarning(source_file, tuple(humanize_validation_error(e)))
+
+
+def humanize_validation_error(error: ValidationError) -> list[str]:
     """Converts a ValidationError to a human-readable format.
 
     This overwrites the default error messages from Pydantic to be better suited for Toolkit users.
@@ -164,24 +144,65 @@ def _humanize_validation_error(error: ValidationError) -> list[str]:
     """
     errors: list[str] = []
     item: ErrorDetails
+
     for item in error.errors(include_input=True, include_url=False):
         loc = item["loc"]
         error_type = item["type"]
+        is_metadata_string_value_error = error_type == "string_type" and len(loc) >= 2 and loc[-2] == "metadata"
         if error_type == "missing":
             msg = f"Missing required field: {loc[-1]!r}"
         elif error_type == "extra_forbidden":
             msg = f"Unused field: {loc[-1]!r}"
         elif error_type == "value_error":
             msg = str(item["ctx"]["error"])
-        elif error_type == "literal_error":
+        elif error_type in {"literal_error", "list_type"}:
             msg = f"{item['msg']}. Got {item['input']!r}."
+        elif is_metadata_string_value_error:
+            # We skip metadata string errors. There are multiple reasons for this
+            # 1. We often allow non-string metadata values, and parse them to string later. For example, in
+            #     ExtractionPipelines.
+            # 2. The user often set metadata values to int/bool/float by mistake, but the server accepts these values and
+            #     converts them to string. Thus, these are not really errors.
+            # 3. The metadata errors flood the output and obscure more important errors, and we see example of
+            #     users ignoring all errors because of this (error fatigue).
+            continue
+        elif error_type == "string_type":
+            msg = f"{item['msg']}. Got {item['input']!r} of type {type(item['input']).__name__}. Hint: Use double quotes to force string."
+        elif error_type == "model_type":
+            model_name = item["ctx"].get("class_name", "unknown")
+            msg = f"Input must be an object of type {model_name}. Got {item['input']!r} of type {type(item['input']).__name__}."
+        elif error_type in {
+            "int_type",
+            "bool_type",
+            "datetime_type",
+            "decimal_type",
+            "float_type",
+            "time_type",
+            "timedelta_type",
+            "dict_type",
+        }:
+            msg = f"{item['msg']}. Got {item['input']!r} of type {type(item['input']).__name__}."
+        elif error_type == "union_tag_not_found" and "ctx" in item and "discriminator" in item["ctx"]:
+            # This is when we use a discriminator field to determine the type in a union. For the user, this means they
+            # are missing a required field.
+            msg = f"Missing required field: {item['ctx']['discriminator']}"
         else:
             # Default to the Pydantic error message
             msg = item["msg"]
-        if len(loc) > 1:
-            msg = f"In {as_json_path(loc[:-1])} {msg[0].casefold()}{msg[1:]}"
+
+        if error_type.endswith("dict_type") and len(loc) > 1:
+            # If this is a dict_type error for a JSON field, the location will be:
+            #  dict[str,json-or-python[json=any,python=tagged-union[list[...],dict[str,...],str,bool,int,float,none]]]
+            #  This is hard to read, so we simplify it to just the field name.
+            loc = tuple(["dict" if isinstance(x, str) and "json-or-python" in x else x for x in loc])
+
+        if len(loc) > 1 and error_type in {"extra_forbidden", "missing"}:
+            # We skip the last element as this is in the message already
+            msg = f"In {as_json_path(loc[:-1])} {msg[:1].casefold()}{msg[1:]}"
+        elif len(loc) > 1:
+            msg = f"In {as_json_path(loc)} {msg[:1].casefold()}{msg[1:]}"
         elif len(loc) == 1 and isinstance(loc[0], str) and error_type not in {"extra_forbidden", "missing"}:
-            msg = f"In field {loc[0]} {msg[0].casefold()}{msg[1:]}"
+            msg = f"In field {loc[0]} {msg[:1].casefold()}{msg[1:]}"
         errors.append(msg)
     return errors
 
@@ -195,8 +216,79 @@ def as_json_path(loc: tuple[str | int, ...]) -> str:
     Returns:
         A JSON path string.
     """
+    if not loc:
+        return ""
     # +1 to convert from 0-based to 1-based indexing
-    if len(loc) == 1 and isinstance(loc[0], int):
-        return f"item [{loc[0] + 1}]"
+    prefix = ""
+    if isinstance(loc[0], int):
+        prefix = "item "
+
+    suffix = ".".join([str(x) if isinstance(x, str) else f"[{x + 1}]" for x in loc]).replace(".[", "[")
+    return f"{prefix}{suffix}"
 
-    return ".".join([str(x) if isinstance(x, str) else f"[{x + 1}]" for x in loc]).replace(".[", "[")
+
+def validate_module_selection(
+    modules: ModuleDirectories,
+    config: BuildConfigYAML,
+    packages: dict[str, list[str]],
+    selected_modules: set[str | Path],
+    organization_dir: Path,
+) -> WarningList:
+    """Validates module selection and returns warnings for non-critical issues.
+
+    Critical errors (duplicate modules, missing modules, no modules selected) are still raised
+    as exceptions as they prevent the build from proceeding.
+    """
+    warnings: WarningList = WarningList()
+
+    # Validations: Ambiguous selection.
+    selected_names = {s for s in config.environment.selected if isinstance(s, str)}
+    if duplicate_modules := {
+        module_name: paths
+        for module_name, paths in modules.as_path_by_name().items()
+        if len(paths) > 1 and module_name in selected_names
+    }:
+        # If the user has selected a module by name, and there are multiple modules with that name, raise an error.
+        # Note, if the user uses a path to select a module, this error will not be raised.
+        raise ToolkitDuplicatedModuleError(
+            f"Ambiguous module selected in config.{config.environment.name}.yaml:", duplicate_modules
+        )
+
+    # Package Referenced Modules Exists
+    for package, package_modules in packages.items():
+        if package not in selected_names:
+            # We do not check packages that are not selected.
+            # Typically, the user will delete the modules that are irrelevant for them;
+            # thus we only check the selected packages.
+            continue
+        if missing_packages := set(package_modules) - modules.available_names:
+            raise ToolkitMissingModuleError(
+                f"Package {package} defined in {CDFToml.file_name!s} is referring "
+                f"the following missing modules {missing_packages}."
+            )
+
+    # Selected modules does not exists
+    if missing_modules := set(selected_modules) - modules.available:
+        hint = ModuleDefinition.long(missing_modules, organization_dir)
+        raise ToolkitMissingModuleError(
+            f"The following selected modules are missing, please check path: {missing_modules}.\n{hint}"
+        )
+
+    # Nothing is Selected
+    if not modules.selected:
+        raise ToolkitEnvError(
+            f"No selected modules specified in {config.filepath!s}, have you configured "
+            f"the environment ({config.environment.name})?"
+        )
+
+    # Dev modules warning (non-critical)
+    dev_modules = modules.available_names & DEV_ONLY_MODULES
+    if dev_modules and config.environment.validation_type != "dev":
+        warnings.append(
+            MediumSeverityWarning(
+                "The following modules should [bold]only[/bold] be used a in CDF Projects designated as dev (development): "
+                f"{humanize_collection(dev_modules)!r}",
+            )
+        )
+
+    return warnings

cognite_toolkit/_repo_files/.gitignore

@@ -174,6 +174,7 @@ celerybeat.pid
 
 # Environments
 .env
+*.env
 .venv
 .local/
 env/
@@ -276,3 +277,4 @@ $RECYCLE.BIN/
 build.*
 build/
 function_local_venvs/
+data/

cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml

@@ -26,4 +26,4 @@ jobs:
       - script: cdf build
         displayName: 'Build the modules'
       - script: cdf deploy
-        displayName: 'Deploy the modules'
+        displayName: 'Deploy the modules'

cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml

@@ -28,4 +28,4 @@ jobs:
       - script: cdf build
         displayName: 'Build the modules'
       - script: cdf deploy --dry-run
-        displayName: 'Dry Run the modules deployment'
+        displayName: 'Dry Run the modules deployment'

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.5.44
+      image: cognite/toolkit:0.7.34
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.5.44
+      image: cognite/toolkit:0.7.34
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -0,0 +1,18 @@
+[cdf]#<PLACEHOLDER>
+default_env = "<DEFAULT_ENV_PLACEHOLDER>"
+
+[modules]
+# This is the version of the modules. It should not be changed manually.
+# It will be updated by the 'cdf modules upgrade' command.
+version = "0.7.34"
+
+
+[plugins]
+dump = false
+dev = false
+data = false
+
+
+[library.toolkit-data]
+url = "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip" 
+checksum = "sha256:b70c207a78d6c94e0977a733571420b250ca641ac8802d8310f1ffe05bf7b2eb"

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.5.44"
+__version__ = "0.7.34"

cognite_toolkit/config.dev.yaml

@@ -0,0 +1,13 @@
+environment:
+  name: dev
+  project: infield-experimental-test-aws
+  validation-type: dev
+  selected:
+    - modules/eriks-location
+
+variables:
+  modules:
+    eriks-location:
+      - location: erik-infield-cdm2
+        location_name: Oslo CDM - Erik tookit
+        location_description: Used to test toolkit ingestion for InField cdm

cognite_toolkit/demo/_base.py

@@ -11,8 +11,8 @@ from rich.panel import Panel
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.commands import AuthCommand, BuildCommand, DeployCommand, ModulesCommand
 from cognite_toolkit._cdf_tk.constants import _RUNNING_IN_BROWSER, MODULES
+from cognite_toolkit._cdf_tk.cruds import CRUDS_BY_FOLDER_NAME
 from cognite_toolkit._cdf_tk.exceptions import AuthenticationError
-from cognite_toolkit._cdf_tk.loaders import LOADER_BY_FOLDER_NAME
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 
 
@@ -98,7 +98,7 @@ class CogniteToolkitDemo:
             if cluster is None:
                 raise ValueError("CDF_CLUSTER is not set in the environment.")
             self._env_vars = EnvironmentVariables(
-                PROVIDER="cdf",  # type: ignore[arg-type]
+                PROVIDER="cdf",
                 LOGIN_FLOW="client_credentials",
                 CDF_CLUSTER=cluster,
                 CDF_PROJECT=self._client.config.project,
@@ -157,7 +157,7 @@ class CogniteToolkitDemo:
             raise AuthenticationError("Environment variables not set.")
 
         deploy.run(
-            lambda: deploy.execute(
+            lambda: deploy.deploy_build_directory(
                 # MyPy fails to see th check above.
                 env_vars=self._env_vars,  # type: ignore[arg-type]
                 build_dir=self._build_dir,
@@ -166,7 +166,7 @@ class CogniteToolkitDemo:
                 drop_data=False,
                 drop=False,
                 force_update=False,
-                include=list(LOADER_BY_FOLDER_NAME.keys()),
+                include=list(CRUDS_BY_FOLDER_NAME.keys()),
                 verbose=False,
             )
         )

{cognite_toolkit-0.5.44.dist-info → cognite_toolkit-0.7.34.dist-info}/METADATA

@@ -1,34 +1,38 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.5.44
+Version: 0.7.34
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
-Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
-Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases
-Project-URL: GitHub, https://github.com/cognitedata/toolkit
-Project-URL: Documentation, https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library
+Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>
 License-Expression: Apache-2.0
-License-File: LICENSE
-Requires-Python: >=3.10
-Requires-Dist: cognite-sdk<8.0.0,>=7.74.2
-Requires-Dist: mixpanel>=4.10.1
-Requires-Dist: packaging<25.0,>=22.0
-Requires-Dist: pandas<3.0.0,>=1.5.3
-Requires-Dist: pip>=25.0.1
-Requires-Dist: pydantic>=2.11.0
 Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: cognite-sdk>=7.87.0,<8.0.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: pandas>=1.5.3,<3.0.0
 Requires-Dist: pyyaml>=6.0.1
-Requires-Dist: questionary>=2.0.1
+Requires-Dist: typer>=0.12.0,<1.0.0
 Requires-Dist: rich>=13.9.4
-Requires-Dist: sentry-sdk>=2.1.0
-Requires-Dist: toml>=0.10.2
-Requires-Dist: tomli<3.0.0,>=2.0.1; python_version < '3.11'
-Requires-Dist: typer<1.0.0,>=0.12.0
+Requires-Dist: questionary>=2.0.1
+Requires-Dist: tomli>=2.0.1,<3.0.0 ; python_full_version < '3.11'
+Requires-Dist: packaging>=25
 Requires-Dist: typing-extensions>=4.0.0
+Requires-Dist: toml>=0.10.2
+Requires-Dist: sentry-sdk>=2.1.0
+Requires-Dist: mixpanel>=4.10.1
+Requires-Dist: pydantic>=2.11.0
+Requires-Dist: python-dateutil>=2.9.0
+Requires-Dist: pip>=25.0.1
+Requires-Dist: filelock>=3.18.0
+Requires-Dist: sqlparse>=0.5.3 ; extra == 'sql'
+Requires-Dist: pyarrow>=20.0.0 ; extra == 'table'
+Requires-Dist: openpyxl>=3.1.5 ; extra == 'table'
+Requires-Python: >=3.10
+Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases
+Project-URL: Documentation, https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library
+Project-URL: GitHub, https://github.com/cognitedata/toolkit
+Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Provides-Extra: sql
-Requires-Dist: sqlparse>=0.5.3; extra == 'sql'
 Provides-Extra: table
-Requires-Dist: pyarrow>=20.0.0; extra == 'table'
 Description-Content-Type: text/markdown
 
 # Cognite Data Fusion Toolkit
@@ -51,7 +55,7 @@ It supports three different modes of operation:
 1. As an **interactive command-line tool** used alongside the Cognite Data Fusion web application to retrieve and
    push configuration of the different Cognite Data Fusion services like data sets, data models, transformations,
    and more. This mode also supports configuration of new Cognite Data Fusion projects to quickly get started.
-2. As tool to support the **project life-cyle by scripting and automating** configuration and management of Cognite Data
+2. As tool to support the **project life-cycle by scripting and automating** configuration and management of Cognite Data
    Fusion projects where CDF configurations are kept as yaml-files that can be checked into version
    control. This mode also supports DevOps workflows with development, staging, and production projects.
 3. As a **tool to deploy official Cognite project templates** to your Cognite Data Fusion project. The tool comes