cognite-toolkit 0.7.29__tar.gz → 0.7.31__tar.gz

{cognite_toolkit-0.7.29 → cognite_toolkit-0.7.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.29
+Version: 0.7.31
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>

{cognite_toolkit-0.7.29 → cognite_toolkit-0.7.31}/cognite_toolkit/_cdf_tk/apps/_core_app.py

@@ -13,8 +13,10 @@ from rich.panel import Panel
 from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.commands import BuildCommand, CleanCommand, DeployCommand
+from cognite_toolkit._cdf_tk.commands.build_v2.build_cmd import BuildCommand as BuildCommandV2
 from cognite_toolkit._cdf_tk.commands.clean import AVAILABLE_DATA_TYPES
 from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.utils import get_cicd_environment
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._version import __version__ as current_version
@@ -207,7 +209,11 @@ class CoreApp(typer.Typer):
         if exit_on_warning:
             print_warning = False
 
-        cmd = BuildCommand(print_warning=print_warning)
+        cmd = (
+            BuildCommandV2(print_warning=print_warning)
+            if Flags.v08.is_enabled()
+            else BuildCommand(print_warning=print_warning)
+        )
         cmd.run(
             lambda: cmd.execute(
                 verbose,

{cognite_toolkit-0.7.29 → cognite_toolkit-0.7.31}/cognite_toolkit/_cdf_tk/commands/__init__.py

@@ -24,6 +24,7 @@ __all__ = [
     "AboutCommand",
     "AuthCommand",
     "BuildCommand",
+    "BuildCommandV2",
     "CleanCommand",
     "CollectCommand",
     "DeployCommand",

{cognite_toolkit-0.7.29 → cognite_toolkit-0.7.31}/cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -6,7 +6,6 @@ from pathlib import Path
 from cognite.client.data_classes.data_modeling import (
     ViewId,
 )
-from pydantic import ValidationError
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
@@ -20,18 +19,15 @@ from cognite_toolkit._cdf_tk.storageio import (
     get_upload_io,
 )
 from cognite_toolkit._cdf_tk.storageio._base import TableUploadableStorageIO, UploadItem
-from cognite_toolkit._cdf_tk.storageio.selectors import Selector, SelectorAdapter
+from cognite_toolkit._cdf_tk.storageio.selectors import Selector, load_selector
 from cognite_toolkit._cdf_tk.storageio.selectors._instances import InstanceSpaceSelector
-from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
-from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning, ToolkitWarning
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
-from cognite_toolkit._cdf_tk.utils.file import read_yaml_file
 from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, ItemMessage, SuccessResponseItems
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
 from cognite_toolkit._cdf_tk.utils.progress_tracker import ProgressTracker
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
-from cognite_toolkit._cdf_tk.validation import humanize_validation_error
 
 from ._base import ToolkitCommand
 from .deploy import DeployCommand
@@ -141,17 +137,11 @@ class UploadCommand(ToolkitCommand):
         """Finds data files and their corresponding metadata files in the input directory."""
         data_files_by_metadata: dict[Selector, list[Path]] = {}
         for manifest_file in input_dir.glob(f"*{DATA_MANIFEST_SUFFIX}"):
-            selector_dict = read_yaml_file(manifest_file, expected_output="dict")
-            try:
-                selector = SelectorAdapter.validate_python(selector_dict)
-            except ValidationError as e:
-                errors = humanize_validation_error(e)
-                self.warn(
-                    ResourceFormatWarning(
-                        manifest_file, tuple(errors), text="Invalid selector in metadata file, skipping."
-                    )
-                )
+            selector_or_warning = load_selector(manifest_file)
+            if isinstance(selector_or_warning, ToolkitWarning):
+                self.warn(selector_or_warning)
                 continue
+            selector: Selector = selector_or_warning
             data_files = selector.find_data_files(input_dir, manifest_file)
             if not data_files:
                 self.warn(

{cognite_toolkit-0.7.29 → cognite_toolkit-0.7.31}/cognite_toolkit/_cdf_tk/commands/build_cmd.py

@@ -20,7 +20,6 @@ from cognite_toolkit._cdf_tk.constants import (
     _RUNNING_IN_BROWSER,
     BUILD_FOLDER_ENCODING,
     DEFAULT_ENV,
-    DEV_ONLY_MODULES,
     HINT_LEAD_TEXT,
     ROOT_MODULES,
     TEMPLATE_VARS_FILE_SUFFIXES,
@@ -59,10 +58,7 @@ from cognite_toolkit._cdf_tk.data_classes import (
     SourceLocationLazy,
 )
 from cognite_toolkit._cdf_tk.exceptions import (
-    ToolkitDuplicatedModuleError,
-    ToolkitEnvError,
     ToolkitError,
-    ToolkitMissingModuleError,
     ToolkitYAMLFormatError,
 )
 from cognite_toolkit._cdf_tk.hints import Hint, ModuleDefinition, verify_module_directory
@@ -89,6 +85,7 @@ from cognite_toolkit._cdf_tk.utils.file import safe_rmtree
 from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules
 from cognite_toolkit._cdf_tk.validation import (
     validate_data_set_is_set,
+    validate_module_selection,
     validate_modules_variables,
     validate_resource_yaml_pydantic,
 )
@@ -207,7 +204,11 @@ class BuildCommand(ToolkitCommand):
 
         user_selected_modules = config.environment.get_selected_modules(packages)
         modules = ModuleDirectories.load(organization_dir, user_selected_modules)
-        self._validate_modules(modules, config, packages, user_selected_modules, organization_dir)
+        module_warnings = validate_module_selection(modules, config, packages, user_selected_modules, organization_dir)
+        if module_warnings:
+            self.warning_list.extend(module_warnings)
+            if self.print_warning:
+                print(str(module_warnings))
 
         if verbose:
             self.console("Selected packages:")
@@ -422,62 +423,6 @@ class BuildCommand(ToolkitCommand):
         builder = self._builder_by_resource_folder[resource_name]
         return builder
 
-    def _validate_modules(
-        self,
-        modules: ModuleDirectories,
-        config: BuildConfigYAML,
-        packages: dict[str, list[str]],
-        selected_modules: set[str | Path],
-        organization_dir: Path,
-    ) -> None:
-        # Validations: Ambiguous selection.
-        selected_names = {s for s in config.environment.selected if isinstance(s, str)}
-        if duplicate_modules := {
-            module_name: paths
-            for module_name, paths in modules.as_path_by_name().items()
-            if len(paths) > 1 and module_name in selected_names
-        }:
-            # If the user has selected a module by name, and there are multiple modules with that name, raise an error.
-            # Note, if the user uses a path to select a module, this error will not be raised.
-            raise ToolkitDuplicatedModuleError(
-                f"Ambiguous module selected in config.{config.environment.name}.yaml:", duplicate_modules
-            )
-        # Package Referenced Modules Exists
-        for package, package_modules in packages.items():
-            if package not in selected_names:
-                # We do not check packages that are not selected.
-                # Typically, the user will delete the modules that are irrelevant for them;
-                # thus we only check the selected packages.
-                continue
-            if missing_packages := set(package_modules) - modules.available_names:
-                ToolkitMissingModuleError(
-                    f"Package {package} defined in {CDFToml.file_name!s} is referring "
-                    f"the following missing modules {missing_packages}."
-                )
-
-        # Selected modules does not exists
-        if missing_modules := set(selected_modules) - modules.available:
-            hint = ModuleDefinition.long(missing_modules, organization_dir)
-            raise ToolkitMissingModuleError(
-                f"The following selected modules are missing, please check path: {missing_modules}.\n{hint}"
-            )
-
-        # Nothing is Selected
-        if not modules.selected:
-            raise ToolkitEnvError(
-                f"No selected modules specified in {config.filepath!s}, have you configured "
-                f"the environment ({config.environment.name})?"
-            )
-
-        dev_modules = modules.available_names & DEV_ONLY_MODULES
-        if dev_modules and config.environment.validation_type != "dev":
-            self.warn(
-                MediumSeverityWarning(
-                    "The following modules should [bold]only[/bold] be used a in CDF Projects designated as dev (development): "
-                    f"{humanize_collection(dev_modules)!r}",
-                )
-            )
-
     def _replace_variables(
         self,
         resource_files: Sequence[Path],

cognite_toolkit-0.7.31/cognite_toolkit/_cdf_tk/commands/build_v2/build_cmd.py

@@ -0,0 +1,241 @@
+from pathlib import Path
+from typing import Any, Literal, TypedDict
+
+from rich import print
+from rich.panel import Panel
+
+from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
+from cognite_toolkit._cdf_tk.commands.build_cmd import BuildCommand as OldBuildCommand
+from cognite_toolkit._cdf_tk.commands.build_v2.build_input import BuildInput
+from cognite_toolkit._cdf_tk.commands.build_v2.build_issues import BuildIssue, BuildIssueList
+from cognite_toolkit._cdf_tk.data_classes import (
+    BuildConfigYAML,
+    BuildVariables,
+    BuiltModuleList,
+    ModuleDirectories,
+)
+from cognite_toolkit._cdf_tk.exceptions import ToolkitError
+from cognite_toolkit._cdf_tk.hints import verify_module_directory
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
+from cognite_toolkit._cdf_tk.utils.file import safe_rmtree
+from cognite_toolkit._cdf_tk.validation import validate_module_selection, validate_modules_variables
+from cognite_toolkit._version import __version__
+
+
+class BuildWarnings(TypedDict):
+    warning: ToolkitWarning
+    location: list[Path]
+
+
+class BuildCommand(ToolkitCommand):
+    def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False) -> None:
+        super().__init__(print_warning, skip_tracking, silent)
+        self.issues = BuildIssueList()
+
+    def execute(
+        self,
+        verbose: bool,
+        organization_dir: Path,
+        build_dir: Path,
+        selected: list[str | Path] | None,
+        build_env_name: str | None,
+        no_clean: bool,
+        client: ToolkitClient | None = None,
+        on_error: Literal["continue", "raise"] = "continue",
+    ) -> BuiltModuleList:
+        """
+        Build the resources into deployable artifacts in the build directory.
+        """
+
+        self.verbose = verbose
+        self.on_error = on_error
+
+        # Tracking the project and cluster for the build.
+        if client:
+            self._additional_tracking_info.project = client.config.project
+            self._additional_tracking_info.cluster = client.config.cdf_cluster
+
+        # Setting the parameters for the build.
+        input = BuildInput.load(organization_dir, build_dir, build_env_name, client, selected)
+
+        # Print the build input.
+        if self.verbose:
+            self._print_build_input(input)
+
+        # Capture warnings from module structure integrity
+        if module_selection_issues := self._validate_modules(input):
+            self.issues.extend(module_selection_issues)
+
+        # Logistics: clean and create build directory
+        if prepare_issues := self._prepare_target_directory(input, not no_clean):
+            self.issues.extend(prepare_issues)
+
+        # Compile the configuration and variables,
+        # check syntax on module and resource level
+        # for any "compilation errors and warnings"
+        built_modules, build_integrity_issues = self._build_configuration(input)
+        if build_integrity_issues:
+            self.issues.extend(build_integrity_issues)
+
+        # This is where we would add any recommendations for the user to improve the build.
+        if build_quality_issues := self._verify_build_quality(built_modules):
+            self.issues.extend(build_quality_issues)
+
+        # Finally, print warnings grouped by category/code and location.
+        self._print_or_log_warnings_by_category(self.issues)
+
+        return built_modules
+
+    def _print_build_input(self, input: BuildInput) -> None:
+        print(
+            Panel(
+                f"Building {input.organization_dir!s}:\n  - Toolkit Version '{__version__!s}'\n"
+                f"  - Environment name {input.build_env_name!r}, validation-type {input.config.environment.validation_type!r}.\n"
+                f"  - Config '{input.config.filepath!s}'",
+                expand=False,
+            )
+        )
+
+    def _prepare_target_directory(self, input: BuildInput, clean: bool = False) -> BuildIssueList:
+        """
+        Directory logistics
+        """
+        issues = BuildIssueList()
+        if input.build_dir.exists() and any(input.build_dir.iterdir()):
+            if not clean:
+                raise ToolkitError("Build directory is not empty. Run without --no-clean to remove existing files.")
+
+            if self.verbose:
+                issues.append(BuildIssue(description=f"Build directory {input.build_dir!s} is not empty. Clearing."))
+            safe_rmtree(input.build_dir)
+        input.build_dir.mkdir(parents=True, exist_ok=True)
+        return issues
+
+    def _validate_modules(self, input: BuildInput) -> BuildIssueList:
+        issues = BuildIssueList()
+        # Verify that the modules exists, are not duplicates,
+        # and at least one is selected
+        verify_module_directory(input.organization_dir, input.build_env_name)
+
+        # Validate module selection
+        user_selected_modules = input.config.environment.get_selected_modules({})
+        module_warnings = validate_module_selection(
+            modules=input.modules,
+            config=input.config,
+            packages={},
+            selected_modules=user_selected_modules,
+            organization_dir=input.organization_dir,
+        )
+        if module_warnings:
+            issues.extend(BuildIssueList.from_warning_list(module_warnings))
+
+        # Validate variables. Note: this looks for non-replaced template
+        # variables <.*?> and can be improved in the future.
+        # Keeping for reference.
+        variables_warnings = validate_modules_variables(input.variables, input.config.filepath)
+        if variables_warnings:
+            issues.extend(BuildIssueList.from_warning_list(variables_warnings))
+
+        # Track LOC of managed configuration
+        # Note: _track is not implemented yet, so we skip it for now
+        # self._track(input)
+
+        return issues
+
+    def _build_configuration(self, input: BuildInput) -> tuple[BuiltModuleList, BuildIssueList]:
+        issues = BuildIssueList()
+        # Use input.modules.selected directly (it's already a ModuleDirectories)
+        if not input.modules.selected:
+            return BuiltModuleList(), issues
+
+        # first collect variables into practical lookup
+        # TODO: parallelism is not implemented yet. I'm sure there are optimizations to be had here, but we'll focus on process parallelism since we believe loading yaml and file i/O are the biggest bottlenecks.
+
+        old_build_command = OldBuildCommand(print_warning=False, skip_tracking=False)
+        built_modules = old_build_command.build_config(
+            build_dir=input.build_dir,
+            organization_dir=input.organization_dir,
+            config=input.config,
+            packages={},
+            clean=False,
+            verbose=self.verbose,
+            client=input.client,
+            progress_bar=False,
+            on_error=self.on_error,
+        )
+        # Copy tracking info from old command to self
+        self._additional_tracking_info.package_ids.update(old_build_command._additional_tracking_info.package_ids)
+        self._additional_tracking_info.module_ids.update(old_build_command._additional_tracking_info.module_ids)
+
+        # Collect warnings from the old build command and convert to issues
+        # Always convert warnings to issues, even if the list appears empty
+        # (WarningList might have custom __bool__ behavior)
+        if old_build_command.warning_list:
+            converted_issues = BuildIssueList.from_warning_list(old_build_command.warning_list)
+            issues.extend(converted_issues)
+        return built_modules, issues
+
+    def _verify_build_quality(self, built_modules: BuiltModuleList) -> BuildIssueList:
+        issues = BuildIssueList()
+        return issues
+
+    def _write(self, input: BuildInput) -> None:
+        # Write the build to the build directory.
+        # Track lines of code built.
+        raise NotImplementedError()
+
+    def _track(self, input: BuildInput) -> None:
+        raise NotImplementedError()
+
+    def _print_or_log_warnings_by_category(self, issues: BuildIssueList) -> None:
+        pass
+
+    # Delegate to old BuildCommand for backward compatibility with tests
+    def build_modules(
+        self,
+        modules: ModuleDirectories,
+        build_dir: Path,
+        variables: BuildVariables,
+        verbose: bool = False,
+        progress_bar: bool = False,
+        on_error: Literal["continue", "raise"] = "continue",
+    ) -> BuiltModuleList:
+        """Delegate to old BuildCommand for backward compatibility."""
+        old_cmd = OldBuildCommand()
+
+        built_modules = old_cmd.build_modules(modules, build_dir, variables, verbose, progress_bar, on_error)
+        self._additional_tracking_info.package_ids.update(old_cmd._additional_tracking_info.package_ids)
+        self._additional_tracking_info.module_ids.update(old_cmd._additional_tracking_info.module_ids)
+        self.issues.extend(BuildIssueList.from_warning_list(old_cmd.warning_list or WarningList[ToolkitWarning]()))
+        return built_modules
+
+    def build_config(
+        self,
+        build_dir: Path,
+        organization_dir: Path,
+        config: BuildConfigYAML,
+        packages: dict[str, list[str]],
+        clean: bool = False,
+        verbose: bool = False,
+        client: ToolkitClient | None = None,
+        progress_bar: bool = False,
+        on_error: Literal["continue", "raise"] = "continue",
+    ) -> BuiltModuleList:
+        """Delegate to old BuildCommand for backward compatibility."""
+        old_cmd = OldBuildCommand()
+        return old_cmd.build_config(
+            build_dir, organization_dir, config, packages, clean, verbose, client, progress_bar, on_error
+        )
+
+    def _replace_variables(
+        self,
+        resource_files: list[Path],
+        variables: BuildVariables,
+        resource_name: str,
+        module_dir: Path,
+        verbose: bool = False,
+    ) -> list[Any]:
+        """Delegate to old BuildCommand for backward compatibility."""
+        old_cmd = OldBuildCommand()
+        return old_cmd._replace_variables(resource_files, variables, resource_name, module_dir, verbose)

cognite_toolkit-0.7.31/cognite_toolkit/_cdf_tk/commands/build_v2/build_input.py

@@ -0,0 +1,85 @@
+import sys
+from functools import cached_property
+from pathlib import Path
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from pydantic import BaseModel, ConfigDict
+
+from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.constants import DEFAULT_ENV
+from cognite_toolkit._cdf_tk.data_classes import (
+    BuildConfigYAML,
+    BuildVariables,
+    ModuleDirectories,
+)
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
+from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules
+
+
+class BuildInput(BaseModel):
+    """Input to the build process."""
+
+    # need this until we turn BuildConfigYaml and ToolkitClient into Pydantic models
+    model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True)
+
+    organization_dir: Path
+    build_dir: Path
+    build_env_name: str
+    config: BuildConfigYAML
+    client: ToolkitClient | None = None
+    selected: list[str | Path] | None = None
+    warnings: WarningList[ToolkitWarning] | None = None
+
+    @classmethod
+    def load(
+        cls,
+        organization_dir: Path,
+        build_dir: Path,
+        build_env_name: str | None,
+        client: ToolkitClient | None,
+        selected: list[str | Path] | None = None,
+    ) -> Self:
+        resolved_org_dir = Path.cwd() if organization_dir in {Path("."), Path("./")} else organization_dir
+        resolved_env = build_env_name or DEFAULT_ENV
+        config, warnings = cls._load_config(resolved_org_dir, resolved_env, selected)
+        return cls(
+            organization_dir=resolved_org_dir,
+            build_dir=build_dir,
+            build_env_name=resolved_env,
+            config=config,
+            client=client,
+            selected=selected,
+            warnings=warnings,
+        )
+
+    @classmethod
+    def _load_config(
+        cls, organization_dir: Path, build_env_name: str, selected: list[str | Path] | None
+    ) -> tuple[BuildConfigYAML, WarningList[ToolkitWarning]]:
+        warnings: WarningList[ToolkitWarning] = WarningList[ToolkitWarning]()
+        if (organization_dir / BuildConfigYAML.get_filename(build_env_name or DEFAULT_ENV)).exists():
+            config = BuildConfigYAML.load_from_directory(organization_dir, build_env_name or DEFAULT_ENV)
+        else:
+            # Loads the default environment
+            config = BuildConfigYAML.load_default(organization_dir)
+        if selected:
+            config.environment.selected = parse_user_selected_modules(selected, organization_dir)
+        config.set_environment_variables()
+        if environment_warning := config.validate_environment():
+            warnings.append(environment_warning)
+        return config, warnings
+
+    @cached_property
+    def modules(self) -> ModuleDirectories:
+        user_selected_modules = self.config.environment.get_selected_modules({})
+        return ModuleDirectories.load(self.organization_dir, user_selected_modules)
+
+    @cached_property
+    def variables(self) -> BuildVariables:
+        return BuildVariables.load_raw(
+            self.config.variables, self.modules.available_paths, self.modules.selected.available_paths
+        )

cognite_toolkit-0.7.31/cognite_toolkit/_cdf_tk/commands/build_v2/build_issues.py

@@ -0,0 +1,27 @@
+import sys
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from collections import UserList
+
+from pydantic import BaseModel
+
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
+
+
+class BuildIssue(BaseModel):
+    """Issue with the build. Can have a recommendation for the user to improve the build."""
+
+    description: str
+
+
+class BuildIssueList(UserList[BuildIssue]):
+    """List of build issues."""
+
+    @classmethod
+    def from_warning_list(cls, warning_list: WarningList[ToolkitWarning]) -> Self:
+        """Create a BuildIssueList from a WarningList."""
+        return cls([BuildIssue(description=warning.get_message()) for warning in warning_list])

{cognite_toolkit-0.7.29 → cognite_toolkit-0.7.31}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py

@@ -26,6 +26,8 @@
 # limitations under the License.
 
 
+import random
+import time
 import warnings
 from collections import defaultdict
 from collections.abc import Callable, Hashable, Iterable, Sequence
@@ -427,25 +429,58 @@ class TransformationCRUD(ResourceCRUD[str, TransformationWrite, Transformation])
                     raise error from e
                 raise e
             except CogniteAPIError as e:
-                if "Failed to bind session using nonce for" in e.message and len(chunk) > 1:
-                    MediumSeverityWarning(
-                        f"Failed to create {len(chunk)} transformations in a batch due to nonce binding error. "
-                        "Trying to recover by creating them one by one."
-                    ).print_warning(console=self.console)
-                    # Retry one by one
-                    for item in chunk:
-                        recovered = self._execute_in_batches(items=[item], api_call=api_call)
-                        results.extend(recovered)
-                    if self.console:
-                        self.console.print(
-                            f"  [bold green]RECOVERED:[/] Successfully created {len(chunk)} transformations one by one."
-                        )
+                if "Failed to bind session using nonce" in e.message and len(chunk) > 1:
+                    results.extend(self._execute_one_by_one(chunk, api_call))
                 else:
                     raise
             else:
                 results.extend(chunk_results)
         return results
 
+    def _execute_one_by_one(
+        self,
+        chunk: Sequence[TransformationWrite],
+        api_call: Callable[[Sequence[TransformationWrite]], TransformationList],
+    ) -> TransformationList:
+        MediumSeverityWarning(
+            f"Failed to create {len(chunk)} transformations in a batch due to nonce binding error. "
+            "Trying to recover by creating them one by one."
+        ).print_warning(console=self.client.console)
+        # Retry one by one
+        failed_ids: list[str] = []
+        success_count = 0
+        delay = 0.3
+        self._sleep_with_jitter(delay, delay + 0.3)
+        results = TransformationList([])
+        for item in chunk:
+            try:
+                recovered = api_call([item])
+            except CogniteAPIError as e:
+                if "Failed to bind session using nonce" in e.message:
+                    failed_ids.append(item.external_id or "<missing>")
+                    self._sleep_with_jitter(delay, delay + 0.3)
+                else:
+                    raise
+            else:
+                results.extend(recovered)
+                success_count += 1
+        message = f"  [bold]RECOVERY COMPLETE:[/] Successfully created {success_count:,} transformations"
+        if failed_ids:
+            message += f", failed to create {len(failed_ids):,} transformations: {humanize_collection(failed_ids)}"
+        else:
+            message += "."
+        if failed_ids:
+            HighSeverityWarning(message).print_warning(include_timestamp=True, console=self.client.console)
+        else:
+            self.client.console.print(message)
+        return results
+
+    @staticmethod
+    def _sleep_with_jitter(base_delay: float, max_delay: float) -> None:
+        """Sleeps for a random duration between base_delay and max_delay (inclusive)."""
+        sleep_time = random.uniform(base_delay, max_delay)
+        time.sleep(sleep_time)
+
     def _update_nonce(self, items: Sequence[TransformationWrite]) -> None:
         for item in items:
             if not item.external_id:

{cognite_toolkit-0.7.29 → cognite_toolkit-0.7.31}/cognite_toolkit/_cdf_tk/feature_flags.py

@@ -57,6 +57,10 @@ class Flags(Enum):
         visible=True,
         description="Enables extended download to support downloading file content and datapoints",
     )
+    EXTEND_UPLOAD = FlagMetadata(
+        visible=True,
+        description="Enables extended upload to support uploading individual files",
+    )
 
     def is_enabled(self) -> bool:
         return FeatureFlag.is_enabled(self)