cognite-toolkit 0.7.29__py3-none-any.whl → 0.7.31__py3-none-any.whl

cognite_toolkit/_cdf_tk/apps/_core_app.py

@@ -13,8 +13,10 @@ from rich.panel import Panel
 from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.commands import BuildCommand, CleanCommand, DeployCommand
+from cognite_toolkit._cdf_tk.commands.build_v2.build_cmd import BuildCommand as BuildCommandV2
 from cognite_toolkit._cdf_tk.commands.clean import AVAILABLE_DATA_TYPES
 from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.utils import get_cicd_environment
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._version import __version__ as current_version
@@ -207,7 +209,11 @@ class CoreApp(typer.Typer):
         if exit_on_warning:
             print_warning = False
 
-        cmd = BuildCommand(print_warning=print_warning)
+        cmd = (
+            BuildCommandV2(print_warning=print_warning)
+            if Flags.v08.is_enabled()
+            else BuildCommand(print_warning=print_warning)
+        )
         cmd.run(
             lambda: cmd.execute(
                 verbose,

cognite_toolkit/_cdf_tk/commands/__init__.py

@@ -24,6 +24,7 @@ __all__ = [
     "AboutCommand",
     "AuthCommand",
     "BuildCommand",
+    "BuildCommandV2",
     "CleanCommand",
     "CollectCommand",
     "DeployCommand",

cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -6,7 +6,6 @@ from pathlib import Path
 from cognite.client.data_classes.data_modeling import (
     ViewId,
 )
-from pydantic import ValidationError
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
@@ -20,18 +19,15 @@ from cognite_toolkit._cdf_tk.storageio import (
     get_upload_io,
 )
 from cognite_toolkit._cdf_tk.storageio._base import TableUploadableStorageIO, UploadItem
-from cognite_toolkit._cdf_tk.storageio.selectors import Selector, SelectorAdapter
+from cognite_toolkit._cdf_tk.storageio.selectors import Selector, load_selector
 from cognite_toolkit._cdf_tk.storageio.selectors._instances import InstanceSpaceSelector
-from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
-from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning, ToolkitWarning
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
-from cognite_toolkit._cdf_tk.utils.file import read_yaml_file
 from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, ItemMessage, SuccessResponseItems
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
 from cognite_toolkit._cdf_tk.utils.progress_tracker import ProgressTracker
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
-from cognite_toolkit._cdf_tk.validation import humanize_validation_error
 
 from ._base import ToolkitCommand
 from .deploy import DeployCommand
@@ -141,17 +137,11 @@ class UploadCommand(ToolkitCommand):
         """Finds data files and their corresponding metadata files in the input directory."""
         data_files_by_metadata: dict[Selector, list[Path]] = {}
         for manifest_file in input_dir.glob(f"*{DATA_MANIFEST_SUFFIX}"):
-            selector_dict = read_yaml_file(manifest_file, expected_output="dict")
-            try:
-                selector = SelectorAdapter.validate_python(selector_dict)
-            except ValidationError as e:
-                errors = humanize_validation_error(e)
-                self.warn(
-                    ResourceFormatWarning(
-                        manifest_file, tuple(errors), text="Invalid selector in metadata file, skipping."
-                    )
-                )
+            selector_or_warning = load_selector(manifest_file)
+            if isinstance(selector_or_warning, ToolkitWarning):
+                self.warn(selector_or_warning)
                 continue
+            selector: Selector = selector_or_warning
             data_files = selector.find_data_files(input_dir, manifest_file)
             if not data_files:
                 self.warn(

cognite_toolkit/_cdf_tk/commands/build_cmd.py

@@ -20,7 +20,6 @@ from cognite_toolkit._cdf_tk.constants import (
     _RUNNING_IN_BROWSER,
     BUILD_FOLDER_ENCODING,
     DEFAULT_ENV,
-    DEV_ONLY_MODULES,
     HINT_LEAD_TEXT,
     ROOT_MODULES,
     TEMPLATE_VARS_FILE_SUFFIXES,
@@ -59,10 +58,7 @@ from cognite_toolkit._cdf_tk.data_classes import (
     SourceLocationLazy,
 )
 from cognite_toolkit._cdf_tk.exceptions import (
-    ToolkitDuplicatedModuleError,
-    ToolkitEnvError,
     ToolkitError,
-    ToolkitMissingModuleError,
     ToolkitYAMLFormatError,
 )
 from cognite_toolkit._cdf_tk.hints import Hint, ModuleDefinition, verify_module_directory
@@ -89,6 +85,7 @@ from cognite_toolkit._cdf_tk.utils.file import safe_rmtree
 from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules
 from cognite_toolkit._cdf_tk.validation import (
     validate_data_set_is_set,
+    validate_module_selection,
     validate_modules_variables,
     validate_resource_yaml_pydantic,
 )
@@ -207,7 +204,11 @@ class BuildCommand(ToolkitCommand):
 
         user_selected_modules = config.environment.get_selected_modules(packages)
         modules = ModuleDirectories.load(organization_dir, user_selected_modules)
-        self._validate_modules(modules, config, packages, user_selected_modules, organization_dir)
+        module_warnings = validate_module_selection(modules, config, packages, user_selected_modules, organization_dir)
+        if module_warnings:
+            self.warning_list.extend(module_warnings)
+            if self.print_warning:
+                print(str(module_warnings))
 
         if verbose:
             self.console("Selected packages:")
@@ -422,62 +423,6 @@ class BuildCommand(ToolkitCommand):
         builder = self._builder_by_resource_folder[resource_name]
         return builder
 
-    def _validate_modules(
-        self,
-        modules: ModuleDirectories,
-        config: BuildConfigYAML,
-        packages: dict[str, list[str]],
-        selected_modules: set[str | Path],
-        organization_dir: Path,
-    ) -> None:
-        # Validations: Ambiguous selection.
-        selected_names = {s for s in config.environment.selected if isinstance(s, str)}
-        if duplicate_modules := {
-            module_name: paths
-            for module_name, paths in modules.as_path_by_name().items()
-            if len(paths) > 1 and module_name in selected_names
-        }:
-            # If the user has selected a module by name, and there are multiple modules with that name, raise an error.
-            # Note, if the user uses a path to select a module, this error will not be raised.
-            raise ToolkitDuplicatedModuleError(
-                f"Ambiguous module selected in config.{config.environment.name}.yaml:", duplicate_modules
-            )
-        # Package Referenced Modules Exists
-        for package, package_modules in packages.items():
-            if package not in selected_names:
-                # We do not check packages that are not selected.
-                # Typically, the user will delete the modules that are irrelevant for them;
-                # thus we only check the selected packages.
-                continue
-            if missing_packages := set(package_modules) - modules.available_names:
-                ToolkitMissingModuleError(
-                    f"Package {package} defined in {CDFToml.file_name!s} is referring "
-                    f"the following missing modules {missing_packages}."
-                )
-
-        # Selected modules does not exists
-        if missing_modules := set(selected_modules) - modules.available:
-            hint = ModuleDefinition.long(missing_modules, organization_dir)
-            raise ToolkitMissingModuleError(
-                f"The following selected modules are missing, please check path: {missing_modules}.\n{hint}"
-            )
-
-        # Nothing is Selected
-        if not modules.selected:
-            raise ToolkitEnvError(
-                f"No selected modules specified in {config.filepath!s}, have you configured "
-                f"the environment ({config.environment.name})?"
-            )
-
-        dev_modules = modules.available_names & DEV_ONLY_MODULES
-        if dev_modules and config.environment.validation_type != "dev":
-            self.warn(
-                MediumSeverityWarning(
-                    "The following modules should [bold]only[/bold] be used a in CDF Projects designated as dev (development): "
-                    f"{humanize_collection(dev_modules)!r}",
-                )
-            )
-
     def _replace_variables(
         self,
         resource_files: Sequence[Path],

cognite_toolkit/_cdf_tk/commands/build_v2/build_cmd.py

@@ -0,0 +1,241 @@
+from pathlib import Path
+from typing import Any, Literal, TypedDict
+
+from rich import print
+from rich.panel import Panel
+
+from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
+from cognite_toolkit._cdf_tk.commands.build_cmd import BuildCommand as OldBuildCommand
+from cognite_toolkit._cdf_tk.commands.build_v2.build_input import BuildInput
+from cognite_toolkit._cdf_tk.commands.build_v2.build_issues import BuildIssue, BuildIssueList
+from cognite_toolkit._cdf_tk.data_classes import (
+    BuildConfigYAML,
+    BuildVariables,
+    BuiltModuleList,
+    ModuleDirectories,
+)
+from cognite_toolkit._cdf_tk.exceptions import ToolkitError
+from cognite_toolkit._cdf_tk.hints import verify_module_directory
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
+from cognite_toolkit._cdf_tk.utils.file import safe_rmtree
+from cognite_toolkit._cdf_tk.validation import validate_module_selection, validate_modules_variables
+from cognite_toolkit._version import __version__
+
+
+class BuildWarnings(TypedDict):
+    warning: ToolkitWarning
+    location: list[Path]
+
+
+class BuildCommand(ToolkitCommand):
+    def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False) -> None:
+        super().__init__(print_warning, skip_tracking, silent)
+        self.issues = BuildIssueList()
+
+    def execute(
+        self,
+        verbose: bool,
+        organization_dir: Path,
+        build_dir: Path,
+        selected: list[str | Path] | None,
+        build_env_name: str | None,
+        no_clean: bool,
+        client: ToolkitClient | None = None,
+        on_error: Literal["continue", "raise"] = "continue",
+    ) -> BuiltModuleList:
+        """
+        Build the resources into deployable artifacts in the build directory.
+        """
+
+        self.verbose = verbose
+        self.on_error = on_error
+
+        # Tracking the project and cluster for the build.
+        if client:
+            self._additional_tracking_info.project = client.config.project
+            self._additional_tracking_info.cluster = client.config.cdf_cluster
+
+        # Setting the parameters for the build.
+        input = BuildInput.load(organization_dir, build_dir, build_env_name, client, selected)
+
+        # Print the build input.
+        if self.verbose:
+            self._print_build_input(input)
+
+        # Capture warnings from module structure integrity
+        if module_selection_issues := self._validate_modules(input):
+            self.issues.extend(module_selection_issues)
+
+        # Logistics: clean and create build directory
+        if prepare_issues := self._prepare_target_directory(input, not no_clean):
+            self.issues.extend(prepare_issues)
+
+        # Compile the configuration and variables,
+        # check syntax on module and resource level
+        # for any "compilation errors and warnings"
+        built_modules, build_integrity_issues = self._build_configuration(input)
+        if build_integrity_issues:
+            self.issues.extend(build_integrity_issues)
+
+        # This is where we would add any recommendations for the user to improve the build.
+        if build_quality_issues := self._verify_build_quality(built_modules):
+            self.issues.extend(build_quality_issues)
+
+        # Finally, print warnings grouped by category/code and location.
+        self._print_or_log_warnings_by_category(self.issues)
+
+        return built_modules
+
+    def _print_build_input(self, input: BuildInput) -> None:
+        print(
+            Panel(
+                f"Building {input.organization_dir!s}:\n  - Toolkit Version '{__version__!s}'\n"
+                f"  - Environment name {input.build_env_name!r}, validation-type {input.config.environment.validation_type!r}.\n"
+                f"  - Config '{input.config.filepath!s}'",
+                expand=False,
+            )
+        )
+
+    def _prepare_target_directory(self, input: BuildInput, clean: bool = False) -> BuildIssueList:
+        """
+        Directory logistics
+        """
+        issues = BuildIssueList()
+        if input.build_dir.exists() and any(input.build_dir.iterdir()):
+            if not clean:
+                raise ToolkitError("Build directory is not empty. Run without --no-clean to remove existing files.")
+
+            if self.verbose:
+                issues.append(BuildIssue(description=f"Build directory {input.build_dir!s} is not empty. Clearing."))
+            safe_rmtree(input.build_dir)
+        input.build_dir.mkdir(parents=True, exist_ok=True)
+        return issues
+
+    def _validate_modules(self, input: BuildInput) -> BuildIssueList:
+        issues = BuildIssueList()
+        # Verify that the modules exists, are not duplicates,
+        # and at least one is selected
+        verify_module_directory(input.organization_dir, input.build_env_name)
+
+        # Validate module selection
+        user_selected_modules = input.config.environment.get_selected_modules({})
+        module_warnings = validate_module_selection(
+            modules=input.modules,
+            config=input.config,
+            packages={},
+            selected_modules=user_selected_modules,
+            organization_dir=input.organization_dir,
+        )
+        if module_warnings:
+            issues.extend(BuildIssueList.from_warning_list(module_warnings))
+
+        # Validate variables. Note: this looks for non-replaced template
+        # variables <.*?> and can be improved in the future.
+        # Keeping for reference.
+        variables_warnings = validate_modules_variables(input.variables, input.config.filepath)
+        if variables_warnings:
+            issues.extend(BuildIssueList.from_warning_list(variables_warnings))
+
+        # Track LOC of managed configuration
+        # Note: _track is not implemented yet, so we skip it for now
+        # self._track(input)
+
+        return issues
+
+    def _build_configuration(self, input: BuildInput) -> tuple[BuiltModuleList, BuildIssueList]:
+        issues = BuildIssueList()
+        # Use input.modules.selected directly (it's already a ModuleDirectories)
+        if not input.modules.selected:
+            return BuiltModuleList(), issues
+
+        # first collect variables into practical lookup
+        # TODO: parallelism is not implemented yet. I'm sure there are optimizations to be had here, but we'll focus on process parallelism since we believe loading yaml and file i/O are the biggest bottlenecks.
+
+        old_build_command = OldBuildCommand(print_warning=False, skip_tracking=False)
+        built_modules = old_build_command.build_config(
+            build_dir=input.build_dir,
+            organization_dir=input.organization_dir,
+            config=input.config,
+            packages={},
+            clean=False,
+            verbose=self.verbose,
+            client=input.client,
+            progress_bar=False,
+            on_error=self.on_error,
+        )
+        # Copy tracking info from old command to self
+        self._additional_tracking_info.package_ids.update(old_build_command._additional_tracking_info.package_ids)
+        self._additional_tracking_info.module_ids.update(old_build_command._additional_tracking_info.module_ids)
+
+        # Collect warnings from the old build command and convert to issues
+        # Always convert warnings to issues, even if the list appears empty
+        # (WarningList might have custom __bool__ behavior)
+        if old_build_command.warning_list:
+            converted_issues = BuildIssueList.from_warning_list(old_build_command.warning_list)
+            issues.extend(converted_issues)
+        return built_modules, issues
+
+    def _verify_build_quality(self, built_modules: BuiltModuleList) -> BuildIssueList:
+        issues = BuildIssueList()
+        return issues
+
+    def _write(self, input: BuildInput) -> None:
+        # Write the build to the build directory.
+        # Track lines of code built.
+        raise NotImplementedError()
+
+    def _track(self, input: BuildInput) -> None:
+        raise NotImplementedError()
+
+    def _print_or_log_warnings_by_category(self, issues: BuildIssueList) -> None:
+        pass
+
+    # Delegate to old BuildCommand for backward compatibility with tests
+    def build_modules(
+        self,
+        modules: ModuleDirectories,
+        build_dir: Path,
+        variables: BuildVariables,
+        verbose: bool = False,
+        progress_bar: bool = False,
+        on_error: Literal["continue", "raise"] = "continue",
+    ) -> BuiltModuleList:
+        """Delegate to old BuildCommand for backward compatibility."""
+        old_cmd = OldBuildCommand()
+
+        built_modules = old_cmd.build_modules(modules, build_dir, variables, verbose, progress_bar, on_error)
+        self._additional_tracking_info.package_ids.update(old_cmd._additional_tracking_info.package_ids)
+        self._additional_tracking_info.module_ids.update(old_cmd._additional_tracking_info.module_ids)
+        self.issues.extend(BuildIssueList.from_warning_list(old_cmd.warning_list or WarningList[ToolkitWarning]()))
+        return built_modules
+
+    def build_config(
+        self,
+        build_dir: Path,
+        organization_dir: Path,
+        config: BuildConfigYAML,
+        packages: dict[str, list[str]],
+        clean: bool = False,
+        verbose: bool = False,
+        client: ToolkitClient | None = None,
+        progress_bar: bool = False,
+        on_error: Literal["continue", "raise"] = "continue",
+    ) -> BuiltModuleList:
+        """Delegate to old BuildCommand for backward compatibility."""
+        old_cmd = OldBuildCommand()
+        return old_cmd.build_config(
+            build_dir, organization_dir, config, packages, clean, verbose, client, progress_bar, on_error
+        )
+
+    def _replace_variables(
+        self,
+        resource_files: list[Path],
+        variables: BuildVariables,
+        resource_name: str,
+        module_dir: Path,
+        verbose: bool = False,
+    ) -> list[Any]:
+        """Delegate to old BuildCommand for backward compatibility."""
+        old_cmd = OldBuildCommand()
+        return old_cmd._replace_variables(resource_files, variables, resource_name, module_dir, verbose)

cognite_toolkit/_cdf_tk/commands/build_v2/build_input.py

@@ -0,0 +1,85 @@
+import sys
+from functools import cached_property
+from pathlib import Path
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from pydantic import BaseModel, ConfigDict
+
+from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.constants import DEFAULT_ENV
+from cognite_toolkit._cdf_tk.data_classes import (
+    BuildConfigYAML,
+    BuildVariables,
+    ModuleDirectories,
+)
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
+from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules
+
+
+class BuildInput(BaseModel):
+    """Input to the build process."""
+
+    # need this until we turn BuildConfigYaml and ToolkitClient into Pydantic models
+    model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True)
+
+    organization_dir: Path
+    build_dir: Path
+    build_env_name: str
+    config: BuildConfigYAML
+    client: ToolkitClient | None = None
+    selected: list[str | Path] | None = None
+    warnings: WarningList[ToolkitWarning] | None = None
+
+    @classmethod
+    def load(
+        cls,
+        organization_dir: Path,
+        build_dir: Path,
+        build_env_name: str | None,
+        client: ToolkitClient | None,
+        selected: list[str | Path] | None = None,
+    ) -> Self:
+        resolved_org_dir = Path.cwd() if organization_dir in {Path("."), Path("./")} else organization_dir
+        resolved_env = build_env_name or DEFAULT_ENV
+        config, warnings = cls._load_config(resolved_org_dir, resolved_env, selected)
+        return cls(
+            organization_dir=resolved_org_dir,
+            build_dir=build_dir,
+            build_env_name=resolved_env,
+            config=config,
+            client=client,
+            selected=selected,
+            warnings=warnings,
+        )
+
+    @classmethod
+    def _load_config(
+        cls, organization_dir: Path, build_env_name: str, selected: list[str | Path] | None
+    ) -> tuple[BuildConfigYAML, WarningList[ToolkitWarning]]:
+        warnings: WarningList[ToolkitWarning] = WarningList[ToolkitWarning]()
+        if (organization_dir / BuildConfigYAML.get_filename(build_env_name or DEFAULT_ENV)).exists():
+            config = BuildConfigYAML.load_from_directory(organization_dir, build_env_name or DEFAULT_ENV)
+        else:
+            # Loads the default environment
+            config = BuildConfigYAML.load_default(organization_dir)
+        if selected:
+            config.environment.selected = parse_user_selected_modules(selected, organization_dir)
+        config.set_environment_variables()
+        if environment_warning := config.validate_environment():
+            warnings.append(environment_warning)
+        return config, warnings
+
+    @cached_property
+    def modules(self) -> ModuleDirectories:
+        user_selected_modules = self.config.environment.get_selected_modules({})
+        return ModuleDirectories.load(self.organization_dir, user_selected_modules)
+
+    @cached_property
+    def variables(self) -> BuildVariables:
+        return BuildVariables.load_raw(
+            self.config.variables, self.modules.available_paths, self.modules.selected.available_paths
+        )

cognite_toolkit/_cdf_tk/commands/build_v2/build_issues.py

@@ -0,0 +1,27 @@
+import sys
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from collections import UserList
+
+from pydantic import BaseModel
+
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
+
+
+class BuildIssue(BaseModel):
+    """Issue with the build. Can have a recommendation for the user to improve the build."""
+
+    description: str
+
+
+class BuildIssueList(UserList[BuildIssue]):
+    """List of build issues."""
+
+    @classmethod
+    def from_warning_list(cls, warning_list: WarningList[ToolkitWarning]) -> Self:
+        """Create a BuildIssueList from a WarningList."""
+        return cls([BuildIssue(description=warning.get_message()) for warning in warning_list])

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py

@@ -26,6 +26,8 @@
 # limitations under the License.
 
 
+import random
+import time
 import warnings
 from collections import defaultdict
 from collections.abc import Callable, Hashable, Iterable, Sequence
@@ -427,25 +429,58 @@ class TransformationCRUD(ResourceCRUD[str, TransformationWrite, Transformation])
                     raise error from e
                 raise e
             except CogniteAPIError as e:
-                if "Failed to bind session using nonce for" in e.message and len(chunk) > 1:
-                    MediumSeverityWarning(
-                        f"Failed to create {len(chunk)} transformations in a batch due to nonce binding error. "
-                        "Trying to recover by creating them one by one."
-                    ).print_warning(console=self.console)
-                    # Retry one by one
-                    for item in chunk:
-                        recovered = self._execute_in_batches(items=[item], api_call=api_call)
-                        results.extend(recovered)
-                    if self.console:
-                        self.console.print(
-                            f"  [bold green]RECOVERED:[/] Successfully created {len(chunk)} transformations one by one."
-                        )
+                if "Failed to bind session using nonce" in e.message and len(chunk) > 1:
+                    results.extend(self._execute_one_by_one(chunk, api_call))
                 else:
                     raise
             else:
                 results.extend(chunk_results)
         return results
 
+    def _execute_one_by_one(
+        self,
+        chunk: Sequence[TransformationWrite],
+        api_call: Callable[[Sequence[TransformationWrite]], TransformationList],
+    ) -> TransformationList:
+        MediumSeverityWarning(
+            f"Failed to create {len(chunk)} transformations in a batch due to nonce binding error. "
+            "Trying to recover by creating them one by one."
+        ).print_warning(console=self.client.console)
+        # Retry one by one
+        failed_ids: list[str] = []
+        success_count = 0
+        delay = 0.3
+        self._sleep_with_jitter(delay, delay + 0.3)
+        results = TransformationList([])
+        for item in chunk:
+            try:
+                recovered = api_call([item])
+            except CogniteAPIError as e:
+                if "Failed to bind session using nonce" in e.message:
+                    failed_ids.append(item.external_id or "<missing>")
+                    self._sleep_with_jitter(delay, delay + 0.3)
+                else:
+                    raise
+            else:
+                results.extend(recovered)
+                success_count += 1
+        message = f"  [bold]RECOVERY COMPLETE:[/] Successfully created {success_count:,} transformations"
+        if failed_ids:
+            message += f", failed to create {len(failed_ids):,} transformations: {humanize_collection(failed_ids)}"
+        else:
+            message += "."
+        if failed_ids:
+            HighSeverityWarning(message).print_warning(include_timestamp=True, console=self.client.console)
+        else:
+            self.client.console.print(message)
+        return results
+
+    @staticmethod
+    def _sleep_with_jitter(base_delay: float, max_delay: float) -> None:
+        """Sleeps for a random duration between base_delay and max_delay (inclusive)."""
+        sleep_time = random.uniform(base_delay, max_delay)
+        time.sleep(sleep_time)
+
     def _update_nonce(self, items: Sequence[TransformationWrite]) -> None:
         for item in items:
             if not item.external_id:

cognite_toolkit/_cdf_tk/feature_flags.py

@@ -57,6 +57,10 @@ class Flags(Enum):
         visible=True,
         description="Enables extended download to support downloading file content and datapoints",
     )
+    EXTEND_UPLOAD = FlagMetadata(
+        visible=True,
+        description="Enables extended upload to support uploading individual files",
+    )
 
     def is_enabled(self) -> bool:
         return FeatureFlag.is_enabled(self)

cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py

@@ -1,6 +1,13 @@
+from pathlib import Path
 from typing import Annotated
 
-from pydantic import Field, TypeAdapter
+from pydantic import Field, TypeAdapter, ValidationError
+
+from cognite_toolkit._cdf_tk.feature_flags import Flags
+from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning, ToolkitWarning
+from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+from cognite_toolkit._cdf_tk.utils.file import read_yaml_file
+from cognite_toolkit._cdf_tk.validation import humanize_validation_error
 
 from ._asset_centric import AssetCentricFileSelector, AssetCentricSelector, AssetSubtreeSelector, DataSetSelector
 from ._base import DataSelector
@@ -52,9 +59,33 @@ Selector = Annotated[
     Field(discriminator="type"),
 ]
 
+ALPHA_SELECTORS = {FileIdentifierSelector}
+
 SelectorAdapter: TypeAdapter[Selector] = TypeAdapter(Selector)
 
 
+def load_selector(manifest_file: Path) -> Selector | ToolkitWarning:
+    """Loads a selector from a manifest file.
+
+    Args:
+        manifest_file: Path to the manifest file.
+
+    Returns:
+        A selector object or a toolkit warning if loading fails or the selector is an alpha feature that is not enabled.
+    """
+    selector_dict = read_yaml_file(manifest_file, expected_output="dict")
+    try:
+        selector = SelectorAdapter.validate_python(selector_dict)
+    except ValidationError as e:
+        errors = humanize_validation_error(e)
+        return ResourceFormatWarning(manifest_file, tuple(errors), text="Invalid selector in metadata file, skipping.")
+    if not Flags.EXTEND_UPLOAD.is_enabled() and type(selector) in ALPHA_SELECTORS:
+        return MediumSeverityWarning(
+            f"Selector type '{type(selector).__name__}' in file '{manifest_file}' is in alpha. To enable it set the alpha flag 'extend-upload = true' in your CDF.toml file."
+        )
+    return selector
+
+
 __all__ = [
     "AllChartsSelector",
     "AssetCentricFileSelector",
@@ -89,4 +120,5 @@ __all__ = [
     "Selector",
     "SelectorAdapter",
     "TimeSeriesColumn",
+    "load_selector",
 ]

cognite_toolkit/_cdf_tk/utils/http_client/_client.py

@@ -4,7 +4,7 @@ import sys
 import time
 from collections import deque
 from collections.abc import MutableMapping, Sequence, Set
-from typing import Literal
+from typing import Literal, TypeVar
 
 import httpx
 from cognite.client import global_config
@@ -29,6 +29,11 @@ from cognite_toolkit._cdf_tk.utils.http_client._data_classes2 import (
     FailedRequest2,
     FailedResponse2,
     HTTPResult2,
+    ItemsFailedRequest2,
+    ItemsFailedResponse2,
+    ItemsRequest2,
+    ItemsResultMessage2,
+    ItemsSuccessResponse2,
     RequestMessage2,
     SuccessResponse2,
 )
@@ -41,6 +46,8 @@ else:
 
 from cognite_toolkit._cdf_tk.client.config import ToolkitClientConfig
 
+_T_Request_Message = TypeVar("_T_Request_Message", bound=BaseRequestMessage)
+
 
 class HTTPClient:
     """An HTTP client.
@@ -344,6 +351,17 @@ class HTTPClient:
                 body=response.text,
                 content=response.content,
             )
+        if retry_request := self._retry_request(response, request):
+            return retry_request
+        else:
+            # Permanent failure
+            return FailedResponse2(
+                status_code=response.status_code,
+                body=response.text,
+                error=ErrorDetails2.from_response(response),
+            )
+
+    def _retry_request(self, response: httpx.Response, request: _T_Request_Message) -> _T_Request_Message | None:
         retry_after = self._get_retry_after_in_header(response)
         if retry_after is not None and response.status_code == 429 and request.status_attempt < self._max_retries:
             if self._console is not None:
@@ -359,13 +377,7 @@ class HTTPClient:
             request.status_attempt += 1
             time.sleep(self._backoff_time(request.total_attempts))
             return request
-        else:
-            # Permanent failure
-            return FailedResponse2(
-                status_code=response.status_code,
-                body=response.text,
-                error=ErrorDetails2.from_response(response),
-            )
+        return None
 
     def _handle_error_single(self, e: Exception, request: RequestMessage2) -> RequestMessage2 | HTTPResult2:
         if isinstance(e, httpx.ReadTimeout | httpx.TimeoutException):
@@ -388,3 +400,136 @@ class HTTPClient:
             error_msg = f"RequestException after {request.total_attempts - 1} attempts ({error_type} error): {e!s}"
 
             return FailedRequest2(error=error_msg)
+
+    def request_items(self, message: ItemsRequest2) -> Sequence[ItemsRequest2 | ItemsResultMessage2]:
+        """Send an HTTP request with multiple items and return the response.
+
+        Args:
+            message (ItemsRequest2): The request message to send.
+        Returns:
+            Sequence[ItemsRequest2 | ItemsResultMessage2]: The response message(s). This can also
+                include ItemsRequest2(s) to be retried or split.
+        """
+        if message.tracker and message.tracker.limit_reached():
+            return [
+                ItemsFailedRequest2(
+                    ids=[item.as_id() for item in message.items],
+                    error_message=f"Aborting further splitting of requests after {message.tracker.failed_split_count} failed attempts.",
+                )
+            ]
+        try:
+            response = self._make_request2(message)
+            results = self._handle_items_response(response, message)
+        except Exception as e:
+            results = self._handle_items_error(e, message)
+        return results
+
+    def request_items_retries(self, message: ItemsRequest2) -> Sequence[ItemsResultMessage2]:
+        """Send an HTTP request with multiple items and handle retries.
+
+        This method will keep retrying the request until it either succeeds or
+        exhausts the maximum number of retries.
+
+        Note this method will use the current thread to process all request, thus
+        it is blocking.
+
+        Args:
+            message (ItemsRequest2): The request message to send.
+        Returns:
+            Sequence[ItemsResultMessage2]: The final response message, which can be either successful response or failed request.
+        """
+        if message.total_attempts > 0:
+            raise RuntimeError(f"ItemsRequest2 has already been attempted {message.total_attempts} times.")
+        pending_requests: deque[ItemsRequest2] = deque()
+        pending_requests.append(message)
+        final_responses: list[ItemsResultMessage2] = []
+        while pending_requests:
+            current_request = pending_requests.popleft()
+            results = self.request_items(current_request)
+
+            for result in results:
+                if isinstance(result, ItemsRequest2):
+                    pending_requests.append(result)
+                elif isinstance(result, ItemsResultMessage2):
+                    final_responses.append(result)
+                else:
+                    raise TypeError(f"Unexpected result type: {type(result)}")
+
+        return final_responses
+
+    def _handle_items_response(
+        self, response: httpx.Response, request: ItemsRequest2
+    ) -> Sequence[ItemsRequest2 | ItemsResultMessage2]:
+        if 200 <= response.status_code < 300:
+            return [
+                ItemsSuccessResponse2(
+                    ids=[item.as_id() for item in request.items],
+                    status_code=response.status_code,
+                    body=response.text,
+                    content=response.content,
+                )
+            ]
+        elif len(request.items) > 1 and response.status_code in self._split_items_status_codes:
+            # 4XX: Status there is at least one item that is invalid, split the batch to get all valid items processed
+            # 5xx: Server error, split to reduce the number of items in each request, and count as a status attempt
+            status_attempts = request.status_attempt
+            if 500 <= response.status_code < 600:
+                status_attempts += 1
+            splits = request.split(status_attempts=status_attempts)
+            if splits[0].tracker and splits[0].tracker.limit_reached():
+                return [
+                    ItemsFailedResponse2(
+                        ids=[item.as_id() for item in request.items],
+                        status_code=response.status_code,
+                        body=response.text,
+                        error=ErrorDetails2.from_response(response),
+                    )
+                ]
+            return splits
+
+        if retry_request := self._retry_request(response, request):
+            return [retry_request]
+        else:
+            # Permanent failure
+            return [
+                ItemsFailedResponse2(
+                    ids=[item.as_id() for item in request.items],
+                    status_code=response.status_code,
+                    body=response.text,
+                    error=ErrorDetails2.from_response(response),
+                )
+            ]
+
+    def _handle_items_error(
+        self, e: Exception, request: ItemsRequest2
+    ) -> Sequence[ItemsRequest2 | ItemsResultMessage2]:
+        if isinstance(e, httpx.ReadTimeout | httpx.TimeoutException):
+            error_type = "read"
+            request.read_attempt += 1
+            attempts = request.read_attempt
+        elif isinstance(e, ConnectionError | httpx.ConnectError | httpx.ConnectTimeout):
+            error_type = "connect"
+            request.connect_attempt += 1
+            attempts = request.connect_attempt
+        else:
+            error_msg = f"Unexpected exception: {e!s}"
+            return [
+                ItemsFailedRequest2(
+                    ids=[item.as_id() for item in request.items],
+                    error_message=error_msg,
+                )
+            ]
+
+        if attempts <= self._max_retries:
+            time.sleep(self._backoff_time(request.total_attempts))
+            return [request]
+        else:
+            # We have already incremented the attempt count, so we subtract 1 here
+            error_msg = f"RequestException after {request.total_attempts - 1} attempts ({error_type} error): {e!s}"
+
+            return [
+                ItemsFailedRequest2(
+                    ids=[item.as_id() for item in request.items],
+                    error_message=error_msg,
+                )
+            ]

cognite_toolkit/_cdf_tk/utils/http_client/_data_classes2.py

@@ -1,13 +1,22 @@
 import gzip
+import sys
 from abc import ABC, abstractmethod
+from collections.abc import Hashable
 from typing import Any, Literal
 
 import httpx
 from cognite.client import global_config
-from pydantic import BaseModel, JsonValue, TypeAdapter, model_validator
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, model_validator
+from pydantic.alias_generators import to_camel
 
+from cognite_toolkit._cdf_tk.utils.http_client._tracker import ItemsRequestTracker
 from cognite_toolkit._cdf_tk.utils.useful_types import PrimitiveType
 
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
 
 class HTTPResult2(BaseModel): ...
 
@@ -95,3 +104,84 @@ class RequestMessage2(BaseRequestMessage):
 
 
 _BODY_SERIALIZER = TypeAdapter(dict[str, JsonValue])
+
+
+class ItemsResultMessage2(BaseModel):
+    ids: list[Hashable]
+
+
+class ItemsFailedRequest2(ItemsResultMessage2):
+    error_message: str
+
+
+class ItemsSuccessResponse2(ItemsResultMessage2):
+    status_code: int
+    body: str
+    content: bytes
+
+
+class ItemsFailedResponse2(ItemsResultMessage2):
+    status_code: int
+    error: ErrorDetails2
+    body: str
+
+
+class BaseModelObject(BaseModel):
+    """Base class for all object. This includes resources and nested objects."""
+
+    # We allow extra fields to support forward compatibility.
+    model_config = ConfigDict(alias_generator=to_camel, extra="allow")
+
+    def dump(self, camel_case: bool = True) -> dict[str, Any]:
+        """Dump the resource to a dictionary.
+
+        This is the default serialization method for request resources.
+        """
+        return self.model_dump(mode="json", by_alias=camel_case, exclude_unset=True)
+
+    @classmethod
+    def _load(cls, resource: dict[str, Any]) -> "Self":
+        """Load method to match CogniteResource signature."""
+        return cls.model_validate(resource)
+
+
+class RequestResource(BaseModelObject, ABC):
+    @abstractmethod
+    def as_id(self) -> Hashable: ...
+
+
+def _set_default_tracker(data: dict[str, Any]) -> ItemsRequestTracker:
+    if "tracker" not in data or data["tracker"] is None:
+        return ItemsRequestTracker(data.get("max_failures_before_abort", 50))
+    return data["tracker"]
+
+
+class ItemsRequest2(BaseRequestMessage):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    items: list[RequestResource]
+    extra_body_fields: dict[str, JsonValue] | None = None
+    max_failures_before_abort: int = 50
+    tracker: ItemsRequestTracker = Field(init=False, default_factory=_set_default_tracker, exclude=True)
+
+    @property
+    def content(self) -> str | bytes | None:
+        body: dict[str, JsonValue] = {"items": [item.dump() for item in self.items]}
+        if self.extra_body_fields:
+            body.update(self.extra_body_fields)
+        res = _BODY_SERIALIZER.dump_json(body)
+        if not global_config.disable_gzip and isinstance(res, str):
+            return gzip.compress(res.encode("utf-8"))
+        return res
+
+    def split(self, status_attempts: int) -> list["ItemsRequest2"]:
+        """Split the request into multiple requests with a single item each."""
+        mid = len(self.items) // 2
+        if mid == 0:
+            return [self]
+        self.tracker.register_failure()
+        messages: list[ItemsRequest2] = []
+        for part in (self.items[:mid], self.items[mid:]):
+            new_request = self.model_copy(update={"items": part, "status_attempt": status_attempts})
+            new_request.tracker = self.tracker
+            messages.append(new_request)
+        return messages

cognite_toolkit/_cdf_tk/validation.py

@@ -8,18 +8,29 @@ from cognite.client.utils._text import to_snake_case
 from pydantic import BaseModel, TypeAdapter, ValidationError
 from pydantic_core import ErrorDetails
 
-from cognite_toolkit._cdf_tk.data_classes import BuildVariables
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
+from cognite_toolkit._cdf_tk.constants import DEV_ONLY_MODULES
+from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML, BuildVariables, ModuleDirectories
+from cognite_toolkit._cdf_tk.exceptions import (
+    ToolkitDuplicatedModuleError,
+    ToolkitEnvError,
+    ToolkitMissingModuleError,
+)
+from cognite_toolkit._cdf_tk.hints import ModuleDefinition
 from cognite_toolkit._cdf_tk.resource_classes import BaseModelResource
 from cognite_toolkit._cdf_tk.tk_warnings import (
     DataSetMissingWarning,
+    MediumSeverityWarning,
     TemplateVariableWarning,
     WarningList,
 )
 from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+from cognite_toolkit._cdf_tk.utils import humanize_collection
 
 __all__ = [
     "humanize_validation_error",
     "validate_data_set_is_set",
+    "validate_module_selection",
     "validate_modules_variables",
 ]
 
@@ -210,3 +221,70 @@ def as_json_path(loc: tuple[str | int, ...]) -> str:
 
     suffix = ".".join([str(x) if isinstance(x, str) else f"[{x + 1}]" for x in loc]).replace(".[", "[")
     return f"{prefix}{suffix}"
+
+
+def validate_module_selection(
+    modules: ModuleDirectories,
+    config: BuildConfigYAML,
+    packages: dict[str, list[str]],
+    selected_modules: set[str | Path],
+    organization_dir: Path,
+) -> WarningList:
+    """Validates module selection and returns warnings for non-critical issues.
+
+    Critical errors (duplicate modules, missing modules, no modules selected) are still raised
+    as exceptions as they prevent the build from proceeding.
+    """
+    warnings: WarningList = WarningList()
+
+    # Validations: Ambiguous selection.
+    selected_names = {s for s in config.environment.selected if isinstance(s, str)}
+    if duplicate_modules := {
+        module_name: paths
+        for module_name, paths in modules.as_path_by_name().items()
+        if len(paths) > 1 and module_name in selected_names
+    }:
+        # If the user has selected a module by name, and there are multiple modules with that name, raise an error.
+        # Note, if the user uses a path to select a module, this error will not be raised.
+        raise ToolkitDuplicatedModuleError(
+            f"Ambiguous module selected in config.{config.environment.name}.yaml:", duplicate_modules
+        )
+
+    # Package Referenced Modules Exists
+    for package, package_modules in packages.items():
+        if package not in selected_names:
+            # We do not check packages that are not selected.
+            # Typically, the user will delete the modules that are irrelevant for them;
+            # thus we only check the selected packages.
+            continue
+        if missing_packages := set(package_modules) - modules.available_names:
+            raise ToolkitMissingModuleError(
+                f"Package {package} defined in {CDFToml.file_name!s} is referring "
+                f"the following missing modules {missing_packages}."
+            )
+
+    # Selected modules does not exists
+    if missing_modules := set(selected_modules) - modules.available:
+        hint = ModuleDefinition.long(missing_modules, organization_dir)
+        raise ToolkitMissingModuleError(
+            f"The following selected modules are missing, please check path: {missing_modules}.\n{hint}"
+        )
+
+    # Nothing is Selected
+    if not modules.selected:
+        raise ToolkitEnvError(
+            f"No selected modules specified in {config.filepath!s}, have you configured "
+            f"the environment ({config.environment.name})?"
+        )
+
+    # Dev modules warning (non-critical)
+    dev_modules = modules.available_names & DEV_ONLY_MODULES
+    if dev_modules and config.environment.validation_type != "dev":
+        warnings.append(
+            MediumSeverityWarning(
+                "The following modules should [bold]only[/bold] be used a in CDF Projects designated as dev (development): "
+                f"{humanize_collection(dev_modules)!r}",
+            )
+        )
+
+    return warnings

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.7.29
+      image: cognite/toolkit:0.7.31
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.7.29
+      image: cognite/toolkit:0.7.31
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.29"
+version = "0.7.31"
 
 
 [plugins]

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.7.29"
+__version__ = "0.7.31"

{cognite_toolkit-0.7.29.dist-info → cognite_toolkit-0.7.31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.29
+Version: 0.7.31
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>

{cognite_toolkit-0.7.29.dist-info → cognite_toolkit-0.7.31.dist-info}/RECORD

@@ -3,7 +3,7 @@ cognite_toolkit/_cdf.py,sha256=sefGD2JQuOTBZhEqSj_ECbNZ7nTRN4AwGwX1pSUhoow,5636
 cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/apps/__init__.py,sha256=KKmhbpvPKTwqQS2g_XqAC2yvtPsvdl8wV5TgJA3zqhs,702
 cognite_toolkit/_cdf_tk/apps/_auth_app.py,sha256=ER7uYb3ViwsHMXiQEZpyhwU6TIjKaB9aEy32VI4MPpg,3397
-cognite_toolkit/_cdf_tk/apps/_core_app.py,sha256=YK0MOK7Tv3cDSe5_6o9GtM5n_6sE7I0Wm-Se4eJnyNM,13744
+cognite_toolkit/_cdf_tk/apps/_core_app.py,sha256=BiY7GWInq0INa7uCiGQyt4cBs1Eguyr5BBCW14JHo3I,14018
 cognite_toolkit/_cdf_tk/apps/_data_app.py,sha256=LeplXlxXtyIymRPgbatQrRFodU4VZBFxI0bqDutLSbg,806
 cognite_toolkit/_cdf_tk/apps/_dev_app.py,sha256=FaY67PFdKwdiMKgJbTcjHT1X2Xfbog2PKL6T-kcawyc,2818
 cognite_toolkit/_cdf_tk/apps/_download_app.py,sha256=2nPn9P_9br9poynSpKKSZF7WYTYT--BfxlxXkSEeH-8,41156
@@ -93,7 +93,7 @@ cognite_toolkit/_cdf_tk/client/testing.py,sha256=mXqEXPMZcbETrXBn6D-SiAcjD7xAkuu
 cognite_toolkit/_cdf_tk/client/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/client/utils/_concurrency.py,sha256=3GtQbKDaosyKHEt-KzxKK9Yie4TvZPdoou2vUk6dUa8,2298
 cognite_toolkit/_cdf_tk/client/utils/_http_client.py,sha256=oXNKrIaizG4WiSAhL_kSCHAuL4aaaEhCU4pOJGxh6Xs,483
-cognite_toolkit/_cdf_tk/commands/__init__.py,sha256=wXRbOwMyhGjMj83_bXTOUXtCh70YS4-eiYzzCMN_YOs,1390
+cognite_toolkit/_cdf_tk/commands/__init__.py,sha256=g-k-P7jWUVzDM0iIBlq9kXdAs_dlGji3NxYEEmTv0L8,1412
 cognite_toolkit/_cdf_tk/commands/_base.py,sha256=1gl8Y-yqfedRMfdbwM3iPTIUIZriX1UvC1deLsJSJwM,2667
 cognite_toolkit/_cdf_tk/commands/_changes.py,sha256=sU0KaTtPVSJgAZcaZ1Tkcajj36pmhd13kh7V8QbIED8,22987
 cognite_toolkit/_cdf_tk/commands/_cli_commands.py,sha256=TK6U_rm6VZT_V941kTyHMoulWgJzbDC8YIIQDPJ5x3w,1011
@@ -113,12 +113,16 @@ cognite_toolkit/_cdf_tk/commands/_migrate/selectors.py,sha256=N1H_-rBpPUD6pbrlco
 cognite_toolkit/_cdf_tk/commands/_profile.py,sha256=_4iX3AHAI6eLmRVUlWXCSvVHx1BZW2yDr_i2i9ECg6U,43120
 cognite_toolkit/_cdf_tk/commands/_purge.py,sha256=HqN2T0OrHG53BfQrolYLi0kFa-_KuJjbZldFtQdMdnY,32914
 cognite_toolkit/_cdf_tk/commands/_questionary_style.py,sha256=h-w7fZKkGls3TrzIGBKjsZSGoXJJIYchgD1StfA40r8,806
-cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=jFt4NG2zEcUu8WUCgRQEoYDIs5C70VTVvTKkZ0F5j1A,13534
+cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=SBqevUZjNW_-hGXSI8YzwgQ7AgJH15M0S3avm7Wmqpw,13040
 cognite_toolkit/_cdf_tk/commands/_utils.py,sha256=UxMJW5QYKts4om5n6x2Tq2ihvfO9gWjhQKeqZNFTlKg,402
 cognite_toolkit/_cdf_tk/commands/_virtual_env.py,sha256=GFAid4hplixmj9_HkcXqU5yCLj-fTXm4cloGD6U2swY,2180
 cognite_toolkit/_cdf_tk/commands/about.py,sha256=pEXNdCeJYONOalH8x-7QRsKLgj-9gdIqN16pPxA3bhg,9395
 cognite_toolkit/_cdf_tk/commands/auth.py,sha256=TX_8YuVCjMVIQjEdfBE66bSDrojJhCnxf_jcT3-9wM8,32550
-cognite_toolkit/_cdf_tk/commands/build_cmd.py,sha256=6m-lK0vccje1gaQ_fd68UvA4CbhuBszDapnHwu4VU_U,30897
+cognite_toolkit/_cdf_tk/commands/build_cmd.py,sha256=c_4S9cZZYxXs2gXZVAEFlPRLZQK_hpCzEMNN1Zoua_o,28284
+cognite_toolkit/_cdf_tk/commands/build_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cognite_toolkit/_cdf_tk/commands/build_v2/build_cmd.py,sha256=U69tPVVWGxMD-FwwTf60pE2gL8MgD0J5wcrVWfmHiME,10146
+cognite_toolkit/_cdf_tk/commands/build_v2/build_input.py,sha256=wHTtUl0OrtNadBZm-i9Ju3zjV7Y6Asth4XuLFZ48qww,3197
+cognite_toolkit/_cdf_tk/commands/build_v2/build_issues.py,sha256=xx11FtNbVfgO4mHgyPbe4m1veYZW_1NDgs8J2PKFa58,739
 cognite_toolkit/_cdf_tk/commands/clean.py,sha256=JNPIjNSiOJaP-cUhFzT8H3s30luA9lI39yH18QxHaYM,16603
 cognite_toolkit/_cdf_tk/commands/collect.py,sha256=zBMKhhvjOpuASMnwP0eeHRI02tANcvFEZgv0CQO1ECc,627
 cognite_toolkit/_cdf_tk/commands/deploy.py,sha256=DpA6q0f17qCpsrYUywavvaRjJ-qaIm78ukw8kIfCNbg,23262
@@ -155,7 +159,7 @@ cognite_toolkit/_cdf_tk/cruds/_resource_cruds/robotics.py,sha256=IeDks5yJFta8Tp4
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py,sha256=1GAIn6G3ZC11ZAgRW-HKaJltMhIqSKH6_ttdEMGgFN0,3054
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/three_d_model.py,sha256=5jIRF2JqosVHyAi2Ek6H38K2FWcNqrbPOR6MTSIEAQI,7320
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/timeseries.py,sha256=rMkA78K9CZqUu7YBDiLgnaZgPcIsWdT-J3nB4xwCJWw,23032
-cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py,sha256=ndhoGIC6tBjNmwI0wDfg8v_6dJfCnHUnTienYOtKwC8,33876
+cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py,sha256=oDgscEy6ybc3k1ZDg8S0vky1YnO895fyKV0hcgQM3Nk,35136
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/workflow.py,sha256=dVkf7OpZHV2H7V3zRU4BzUqDxgAudiWBJPvGeydATCg,26035
 cognite_toolkit/_cdf_tk/cruds/_worker.py,sha256=QeZjziBCePilyI4WfTZvLfSHGT3qJGCMhugsoO1SVtU,9285
 cognite_toolkit/_cdf_tk/data_classes/__init__.py,sha256=4zL-zR3lgQTCWfcy28LK0HEcukQOndPEFXVqfYfdKHU,1720
@@ -173,7 +177,7 @@ cognite_toolkit/_cdf_tk/data_classes/_packages.py,sha256=RPAi54GCBwMZFAA4XgIa1rG
 cognite_toolkit/_cdf_tk/data_classes/_tracking_info.py,sha256=LoUX6GC9I3VXOqOpKna8wFDbCdiC_SR4qBX53vabK0Y,2014
 cognite_toolkit/_cdf_tk/data_classes/_yaml_comments.py,sha256=zfuDu9aAsb1ExeZBAJIqVaoqIZ050tO_oh3dApzlDwY,4937
 cognite_toolkit/_cdf_tk/exceptions.py,sha256=xG0jMwi5A20nvPvyo6sCyz_cyKycynPyIzpYiGR4gcU,6064
-cognite_toolkit/_cdf_tk/feature_flags.py,sha256=ESxrtl2hbe0Fr6e1535MAjSIM326Zc6W7d2PtfjAWkk,2071
+cognite_toolkit/_cdf_tk/feature_flags.py,sha256=X2-penlGM4RlFcX3UXTxXeb85eDrV0NiRpUrfvD7aQA,2218
 cognite_toolkit/_cdf_tk/hints.py,sha256=UI1ymi2T5wCcYOpEbKbVaDnlyFReFy8TDtMVt-5E1h8,6493
 cognite_toolkit/_cdf_tk/plugins.py,sha256=0V14rceAWLZQF8iWdyL5QmK7xB796YaDEtb9RIj5AOc,836
 cognite_toolkit/_cdf_tk/protocols.py,sha256=Lc8XnBfmDZN6dwmSopmK7cFE9a9jZ2zdUryEeCXn27I,3052
@@ -247,7 +251,7 @@ cognite_toolkit/_cdf_tk/storageio/_datapoints.py,sha256=xE1YgoP98-mJjIeF5536KwCh
 cognite_toolkit/_cdf_tk/storageio/_file_content.py,sha256=4b1Lr8ZROLZg7MjT7IiCsRhyVAl1KCWtr0fcDtyzk1o,19062
 cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=t9fNpHnT6kCk8LDoPj3qZXmHpyDbPF5BZ6pI8ziTyFw,10810
 cognite_toolkit/_cdf_tk/storageio/_raw.py,sha256=pgZN5MbqDwMZl9Ow1KouDJUO2Ngga8_b6hwv7H31SVQ,5161
-cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py,sha256=VUK1A76zsu4a25A3oaPUrQEEuRcCpUBK6o8UMMKw7qg,2458
+cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py,sha256=zPLiF5iCzvKF1fyAOAnTUS2YqHCDzNHaPg3hobvwDuw,3916
 cognite_toolkit/_cdf_tk/storageio/selectors/_asset_centric.py,sha256=7Iv_ccVX6Vzt3ZLFZ0Er3hN92iEsFTm9wgF-yermOWE,1467
 cognite_toolkit/_cdf_tk/storageio/selectors/_base.py,sha256=hjFkbmNGsK3QIW-jnJV_8YNmvVROERxzG82qIZhU7SM,3065
 cognite_toolkit/_cdf_tk/storageio/selectors/_canvas.py,sha256=E9S-wr-JUqRosI_2cSCfR0tF8MdIFTrMxDItuWRcuO4,597
@@ -281,9 +285,9 @@ cognite_toolkit/_cdf_tk/utils/fileio/_writers.py,sha256=mc23m0kJgl57FUDvwLmS7yR3
 cognite_toolkit/_cdf_tk/utils/graphql_parser.py,sha256=2i2wDjg_Uw3hJ-pHtPK8hczIuCj5atrK8HZbgWJB-Pk,11532
 cognite_toolkit/_cdf_tk/utils/hashing.py,sha256=3NyNfljyYNTqAyAFBd6XlyWaj43jRzENxIuPdOY6nqo,2116
 cognite_toolkit/_cdf_tk/utils/http_client/__init__.py,sha256=G8b7Bg4yIet5R4Igh3dS2SntWzE6I0iTGBeNlNsSxkQ,857
-cognite_toolkit/_cdf_tk/utils/http_client/_client.py,sha256=OrrGq3GjusxPPzhFoW8iyiphpdbWOWAoaYOeOy9kqjQ,16212
+cognite_toolkit/_cdf_tk/utils/http_client/_client.py,sha256=WQFqzjWobjUYEcO1YV9uOMaoRrfi4APuDFrZakD86BQ,22496
 cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py,sha256=8KEDyRRaOLhwN2eA2vaBAzZ__JDUicUDyir6x_PE5lk,14817
-cognite_toolkit/_cdf_tk/utils/http_client/_data_classes2.py,sha256=mUrmTqTm60WP4JpMmNO07-NfgWQM8Wby7OLCo02I71U,3031
+cognite_toolkit/_cdf_tk/utils/http_client/_data_classes2.py,sha256=mg0BfP7Te5TSgljPFWI0atLtIGa9OPb50jEAXpkzVXE,6078
 cognite_toolkit/_cdf_tk/utils/http_client/_exception.py,sha256=fC9oW6BN0HbUe2AkYABMP7Kj0-9dNYXVFBY5RQztq2c,126
 cognite_toolkit/_cdf_tk/utils/http_client/_tracker.py,sha256=EBBnd-JZ7nc_jYNFJokCHN2UZ9sx0McFLZvlceUYYic,1215
 cognite_toolkit/_cdf_tk/utils/interactive_select.py,sha256=dP_ZFHvzQRPQxRt6EzURY3Z3Ld_otJtCz-nGqUNtt1k,35725
@@ -297,20 +301,20 @@ cognite_toolkit/_cdf_tk/utils/text.py,sha256=1-LQMo633_hEhNhishQo7Buj-7np5Pe4qKk
 cognite_toolkit/_cdf_tk/utils/thread_safe_dict.py,sha256=NbRHcZvWpF9xHP5OkOMGFpxrPNbi0Q3Eea6PUNbGlt4,3426
 cognite_toolkit/_cdf_tk/utils/useful_types.py,sha256=oK88W6G_aK3hebORSQKZjWrq7jG-pO2lkLWSWYMlngM,1872
 cognite_toolkit/_cdf_tk/utils/validate_access.py,sha256=1puswcpgEDNCwdk91dhLqCBSu_aaUAd3Hsw21d-YVFs,21955
-cognite_toolkit/_cdf_tk/validation.py,sha256=KFdPgnNIbVM0yjFF0cqmpBB8MI8e-U-YbBYrP4IiClE,8441
+cognite_toolkit/_cdf_tk/validation.py,sha256=ixxYJA96EYZwFeo3vHjGfuHsFbbPgAJ3RQ-Sh0-PMok,11790
 cognite_toolkit/_repo_files/.env.tmpl,sha256=UmgKZVvIp-OzD8oOcYuwb_6c7vSJsqkLhuFaiVgK7RI,972
 cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi6urlpTZ3k,5274
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=FQFw5rkuhjbBY-cBoP38pdrcUZF6qkK2Y9YKbab8LzU,667
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=e87zJqrsOVfo7QqSnIZY5U32szqyvq5Bjq-e_16xHiQ,2430
-cognite_toolkit/_resources/cdf.toml,sha256=tJP7KRNhWsrkXisd58pe5ZlzUBOAfe4zR5TTAP63tzg,475
-cognite_toolkit/_version.py,sha256=XdC0R1utqmwH52az-uz3mk0vLZyV2OyWiin3soRIguo,23
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=Ft9lViXmay2AajHAwOiFbP1BYBIlVHvxu56qFQlN9Zk,667
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=_gR9FBSVup2h_P2iTlKRvnxV5kdRMZFvHLo-fK6o_Zc,2430
+cognite_toolkit/_resources/cdf.toml,sha256=3YM8T2ivUqrfocBzLTYRWCWyeYXyXfM7wRSY1Zxa7k8,475
+cognite_toolkit/_version.py,sha256=CtvyqFRtdJwNvYSDVmaT9l1R18bsaIwflR8r696jttk,23
 cognite_toolkit/config.dev.yaml,sha256=M33FiIKdS3XKif-9vXniQ444GTZ-bLXV8aFH86u9iUQ,332
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.7.29.dist-info/WHEEL,sha256=93kfTGt3a0Dykt_T-gsjtyS5_p8F_d6CE1NwmBOirzo,79
-cognite_toolkit-0.7.29.dist-info/entry_points.txt,sha256=EtZ17K2mUjh-AY0QNU1CPIB_aDSSOdmtNI_4Fj967mA,84
-cognite_toolkit-0.7.29.dist-info/METADATA,sha256=71UXU4S9UVcZVxhDGazAQOCbTSYyQzD6Kj2FkxNJHYs,4507
-cognite_toolkit-0.7.29.dist-info/RECORD,,
+cognite_toolkit-0.7.31.dist-info/WHEEL,sha256=93kfTGt3a0Dykt_T-gsjtyS5_p8F_d6CE1NwmBOirzo,79
+cognite_toolkit-0.7.31.dist-info/entry_points.txt,sha256=EtZ17K2mUjh-AY0QNU1CPIB_aDSSOdmtNI_4Fj967mA,84
+cognite_toolkit-0.7.31.dist-info/METADATA,sha256=lOFzBCcUEoVSZjcxZh685Mb3FTEhTlrZTN6XXlrlJ4A,4507
+cognite_toolkit-0.7.31.dist-info/RECORD,,