cognite-toolkit 0.7.29__tar.gz → 0.7.30__tar.gz

{cognite_toolkit-0.7.29 → cognite_toolkit-0.7.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.29
+Version: 0.7.30
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>

{cognite_toolkit-0.7.29 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -6,7 +6,6 @@ from pathlib import Path
 from cognite.client.data_classes.data_modeling import (
     ViewId,
 )
-from pydantic import ValidationError
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
@@ -20,18 +19,15 @@ from cognite_toolkit._cdf_tk.storageio import (
     get_upload_io,
 )
 from cognite_toolkit._cdf_tk.storageio._base import TableUploadableStorageIO, UploadItem
-from cognite_toolkit._cdf_tk.storageio.selectors import Selector, SelectorAdapter
+from cognite_toolkit._cdf_tk.storageio.selectors import Selector, load_selector
 from cognite_toolkit._cdf_tk.storageio.selectors._instances import InstanceSpaceSelector
-from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
-from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning, ToolkitWarning
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
-from cognite_toolkit._cdf_tk.utils.file import read_yaml_file
 from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, ItemMessage, SuccessResponseItems
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
 from cognite_toolkit._cdf_tk.utils.progress_tracker import ProgressTracker
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
-from cognite_toolkit._cdf_tk.validation import humanize_validation_error
 
 from ._base import ToolkitCommand
 from .deploy import DeployCommand
@@ -141,17 +137,11 @@ class UploadCommand(ToolkitCommand):
         """Finds data files and their corresponding metadata files in the input directory."""
         data_files_by_metadata: dict[Selector, list[Path]] = {}
         for manifest_file in input_dir.glob(f"*{DATA_MANIFEST_SUFFIX}"):
-            selector_dict = read_yaml_file(manifest_file, expected_output="dict")
-            try:
-                selector = SelectorAdapter.validate_python(selector_dict)
-            except ValidationError as e:
-                errors = humanize_validation_error(e)
-                self.warn(
-                    ResourceFormatWarning(
-                        manifest_file, tuple(errors), text="Invalid selector in metadata file, skipping."
-                    )
-                )
+            selector_or_warning = load_selector(manifest_file)
+            if isinstance(selector_or_warning, ToolkitWarning):
+                self.warn(selector_or_warning)
                 continue
+            selector: Selector = selector_or_warning
             data_files = selector.find_data_files(input_dir, manifest_file)
             if not data_files:
                 self.warn(

{cognite_toolkit-0.7.29 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/commands/build_cmd.py

@@ -20,7 +20,6 @@ from cognite_toolkit._cdf_tk.constants import (
     _RUNNING_IN_BROWSER,
     BUILD_FOLDER_ENCODING,
     DEFAULT_ENV,
-    DEV_ONLY_MODULES,
     HINT_LEAD_TEXT,
     ROOT_MODULES,
     TEMPLATE_VARS_FILE_SUFFIXES,
@@ -59,10 +58,7 @@ from cognite_toolkit._cdf_tk.data_classes import (
     SourceLocationLazy,
 )
 from cognite_toolkit._cdf_tk.exceptions import (
-    ToolkitDuplicatedModuleError,
-    ToolkitEnvError,
     ToolkitError,
-    ToolkitMissingModuleError,
     ToolkitYAMLFormatError,
 )
 from cognite_toolkit._cdf_tk.hints import Hint, ModuleDefinition, verify_module_directory
@@ -89,6 +85,7 @@ from cognite_toolkit._cdf_tk.utils.file import safe_rmtree
 from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules
 from cognite_toolkit._cdf_tk.validation import (
     validate_data_set_is_set,
+    validate_module_selection,
     validate_modules_variables,
     validate_resource_yaml_pydantic,
 )
@@ -207,7 +204,11 @@ class BuildCommand(ToolkitCommand):
 
         user_selected_modules = config.environment.get_selected_modules(packages)
         modules = ModuleDirectories.load(organization_dir, user_selected_modules)
-        self._validate_modules(modules, config, packages, user_selected_modules, organization_dir)
+        module_warnings = validate_module_selection(modules, config, packages, user_selected_modules, organization_dir)
+        if module_warnings:
+            self.warning_list.extend(module_warnings)
+            if self.print_warning:
+                print(str(module_warnings))
 
         if verbose:
             self.console("Selected packages:")
@@ -422,62 +423,6 @@ class BuildCommand(ToolkitCommand):
         builder = self._builder_by_resource_folder[resource_name]
         return builder
 
-    def _validate_modules(
-        self,
-        modules: ModuleDirectories,
-        config: BuildConfigYAML,
-        packages: dict[str, list[str]],
-        selected_modules: set[str | Path],
-        organization_dir: Path,
-    ) -> None:
-        # Validations: Ambiguous selection.
-        selected_names = {s for s in config.environment.selected if isinstance(s, str)}
-        if duplicate_modules := {
-            module_name: paths
-            for module_name, paths in modules.as_path_by_name().items()
-            if len(paths) > 1 and module_name in selected_names
-        }:
-            # If the user has selected a module by name, and there are multiple modules with that name, raise an error.
-            # Note, if the user uses a path to select a module, this error will not be raised.
-            raise ToolkitDuplicatedModuleError(
-                f"Ambiguous module selected in config.{config.environment.name}.yaml:", duplicate_modules
-            )
-        # Package Referenced Modules Exists
-        for package, package_modules in packages.items():
-            if package not in selected_names:
-                # We do not check packages that are not selected.
-                # Typically, the user will delete the modules that are irrelevant for them;
-                # thus we only check the selected packages.
-                continue
-            if missing_packages := set(package_modules) - modules.available_names:
-                ToolkitMissingModuleError(
-                    f"Package {package} defined in {CDFToml.file_name!s} is referring "
-                    f"the following missing modules {missing_packages}."
-                )
-
-        # Selected modules does not exists
-        if missing_modules := set(selected_modules) - modules.available:
-            hint = ModuleDefinition.long(missing_modules, organization_dir)
-            raise ToolkitMissingModuleError(
-                f"The following selected modules are missing, please check path: {missing_modules}.\n{hint}"
-            )
-
-        # Nothing is Selected
-        if not modules.selected:
-            raise ToolkitEnvError(
-                f"No selected modules specified in {config.filepath!s}, have you configured "
-                f"the environment ({config.environment.name})?"
-            )
-
-        dev_modules = modules.available_names & DEV_ONLY_MODULES
-        if dev_modules and config.environment.validation_type != "dev":
-            self.warn(
-                MediumSeverityWarning(
-                    "The following modules should [bold]only[/bold] be used a in CDF Projects designated as dev (development): "
-                    f"{humanize_collection(dev_modules)!r}",
-                )
-            )
-
     def _replace_variables(
         self,
         resource_files: Sequence[Path],

{cognite_toolkit-0.7.29 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/feature_flags.py

@@ -57,6 +57,10 @@ class Flags(Enum):
         visible=True,
         description="Enables extended download to support downloading file content and datapoints",
     )
+    EXTEND_UPLOAD = FlagMetadata(
+        visible=True,
+        description="Enables extended upload to support uploading individual files",
+    )
 
     def is_enabled(self) -> bool:
         return FeatureFlag.is_enabled(self)

{cognite_toolkit-0.7.29 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py

@@ -1,6 +1,13 @@
+from pathlib import Path
 from typing import Annotated
 
-from pydantic import Field, TypeAdapter
+from pydantic import Field, TypeAdapter, ValidationError
+
+from cognite_toolkit._cdf_tk.feature_flags import Flags
+from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning, ToolkitWarning
+from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+from cognite_toolkit._cdf_tk.utils.file import read_yaml_file
+from cognite_toolkit._cdf_tk.validation import humanize_validation_error
 
 from ._asset_centric import AssetCentricFileSelector, AssetCentricSelector, AssetSubtreeSelector, DataSetSelector
 from ._base import DataSelector
@@ -52,9 +59,33 @@ Selector = Annotated[
     Field(discriminator="type"),
 ]
 
+ALPHA_SELECTORS = {FileIdentifierSelector}
+
 SelectorAdapter: TypeAdapter[Selector] = TypeAdapter(Selector)
 
 
+def load_selector(manifest_file: Path) -> Selector | ToolkitWarning:
+    """Loads a selector from a manifest file.
+
+    Args:
+        manifest_file: Path to the manifest file.
+
+    Returns:
+        A selector object or a toolkit warning if loading fails or the selector is an alpha feature that is not enabled.
+    """
+    selector_dict = read_yaml_file(manifest_file, expected_output="dict")
+    try:
+        selector = SelectorAdapter.validate_python(selector_dict)
+    except ValidationError as e:
+        errors = humanize_validation_error(e)
+        return ResourceFormatWarning(manifest_file, tuple(errors), text="Invalid selector in metadata file, skipping.")
+    if not Flags.EXTEND_UPLOAD.is_enabled() and type(selector) in ALPHA_SELECTORS:
+        return MediumSeverityWarning(
+            f"Selector type '{type(selector).__name__}' in file '{manifest_file}' is in alpha. To enable it set the alpha flag 'extend-upload = true' in your CDF.toml file."
+        )
+    return selector
+
+
 __all__ = [
     "AllChartsSelector",
     "AssetCentricFileSelector",
@@ -89,4 +120,5 @@ __all__ = [
     "Selector",
     "SelectorAdapter",
     "TimeSeriesColumn",
+    "load_selector",
 ]

{cognite_toolkit-0.7.29 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/utils/http_client/_client.py

@@ -4,7 +4,7 @@ import sys
 import time
 from collections import deque
 from collections.abc import MutableMapping, Sequence, Set
-from typing import Literal
+from typing import Literal, TypeVar
 
 import httpx
 from cognite.client import global_config
@@ -29,6 +29,11 @@ from cognite_toolkit._cdf_tk.utils.http_client._data_classes2 import (
     FailedRequest2,
     FailedResponse2,
     HTTPResult2,
+    ItemsFailedRequest2,
+    ItemsFailedResponse2,
+    ItemsRequest2,
+    ItemsResultMessage2,
+    ItemsSuccessResponse2,
     RequestMessage2,
     SuccessResponse2,
 )
@@ -41,6 +46,8 @@ else:
 
 from cognite_toolkit._cdf_tk.client.config import ToolkitClientConfig
 
+_T_Request_Message = TypeVar("_T_Request_Message", bound=BaseRequestMessage)
+
 
 class HTTPClient:
     """An HTTP client.
@@ -344,6 +351,17 @@ class HTTPClient:
                 body=response.text,
                 content=response.content,
             )
+        if retry_request := self._retry_request(response, request):
+            return retry_request
+        else:
+            # Permanent failure
+            return FailedResponse2(
+                status_code=response.status_code,
+                body=response.text,
+                error=ErrorDetails2.from_response(response),
+            )
+
+    def _retry_request(self, response: httpx.Response, request: _T_Request_Message) -> _T_Request_Message | None:
         retry_after = self._get_retry_after_in_header(response)
         if retry_after is not None and response.status_code == 429 and request.status_attempt < self._max_retries:
             if self._console is not None:
@@ -359,13 +377,7 @@ class HTTPClient:
             request.status_attempt += 1
             time.sleep(self._backoff_time(request.total_attempts))
             return request
-        else:
-            # Permanent failure
-            return FailedResponse2(
-                status_code=response.status_code,
-                body=response.text,
-                error=ErrorDetails2.from_response(response),
-            )
+        return None
 
     def _handle_error_single(self, e: Exception, request: RequestMessage2) -> RequestMessage2 | HTTPResult2:
         if isinstance(e, httpx.ReadTimeout | httpx.TimeoutException):
@@ -388,3 +400,136 @@ class HTTPClient:
             error_msg = f"RequestException after {request.total_attempts - 1} attempts ({error_type} error): {e!s}"
 
             return FailedRequest2(error=error_msg)
+
+    def request_items(self, message: ItemsRequest2) -> Sequence[ItemsRequest2 | ItemsResultMessage2]:
+        """Send an HTTP request with multiple items and return the response.
+
+        Args:
+            message (ItemsRequest2): The request message to send.
+        Returns:
+            Sequence[ItemsRequest2 | ItemsResultMessage2]: The response message(s). This can also
+                include ItemsRequest2(s) to be retried or split.
+        """
+        if message.tracker and message.tracker.limit_reached():
+            return [
+                ItemsFailedRequest2(
+                    ids=[item.as_id() for item in message.items],
+                    error_message=f"Aborting further splitting of requests after {message.tracker.failed_split_count} failed attempts.",
+                )
+            ]
+        try:
+            response = self._make_request2(message)
+            results = self._handle_items_response(response, message)
+        except Exception as e:
+            results = self._handle_items_error(e, message)
+        return results
+
+    def request_items_retries(self, message: ItemsRequest2) -> Sequence[ItemsResultMessage2]:
+        """Send an HTTP request with multiple items and handle retries.
+
+        This method will keep retrying the request until it either succeeds or
+        exhausts the maximum number of retries.
+
+        Note this method will use the current thread to process all request, thus
+        it is blocking.
+
+        Args:
+            message (ItemsRequest2): The request message to send.
+        Returns:
+            Sequence[ItemsResultMessage2]: The final response message, which can be either successful response or failed request.
+        """
+        if message.total_attempts > 0:
+            raise RuntimeError(f"ItemsRequest2 has already been attempted {message.total_attempts} times.")
+        pending_requests: deque[ItemsRequest2] = deque()
+        pending_requests.append(message)
+        final_responses: list[ItemsResultMessage2] = []
+        while pending_requests:
+            current_request = pending_requests.popleft()
+            results = self.request_items(current_request)
+
+            for result in results:
+                if isinstance(result, ItemsRequest2):
+                    pending_requests.append(result)
+                elif isinstance(result, ItemsResultMessage2):
+                    final_responses.append(result)
+                else:
+                    raise TypeError(f"Unexpected result type: {type(result)}")
+
+        return final_responses
+
+    def _handle_items_response(
+        self, response: httpx.Response, request: ItemsRequest2
+    ) -> Sequence[ItemsRequest2 | ItemsResultMessage2]:
+        if 200 <= response.status_code < 300:
+            return [
+                ItemsSuccessResponse2(
+                    ids=[item.as_id() for item in request.items],
+                    status_code=response.status_code,
+                    body=response.text,
+                    content=response.content,
+                )
+            ]
+        elif len(request.items) > 1 and response.status_code in self._split_items_status_codes:
+            # 4XX: Status there is at least one item that is invalid, split the batch to get all valid items processed
+            # 5xx: Server error, split to reduce the number of items in each request, and count as a status attempt
+            status_attempts = request.status_attempt
+            if 500 <= response.status_code < 600:
+                status_attempts += 1
+            splits = request.split(status_attempts=status_attempts)
+            if splits[0].tracker and splits[0].tracker.limit_reached():
+                return [
+                    ItemsFailedResponse2(
+                        ids=[item.as_id() for item in request.items],
+                        status_code=response.status_code,
+                        body=response.text,
+                        error=ErrorDetails2.from_response(response),
+                    )
+                ]
+            return splits
+
+        if retry_request := self._retry_request(response, request):
+            return [retry_request]
+        else:
+            # Permanent failure
+            return [
+                ItemsFailedResponse2(
+                    ids=[item.as_id() for item in request.items],
+                    status_code=response.status_code,
+                    body=response.text,
+                    error=ErrorDetails2.from_response(response),
+                )
+            ]
+
+    def _handle_items_error(
+        self, e: Exception, request: ItemsRequest2
+    ) -> Sequence[ItemsRequest2 | ItemsResultMessage2]:
+        if isinstance(e, httpx.ReadTimeout | httpx.TimeoutException):
+            error_type = "read"
+            request.read_attempt += 1
+            attempts = request.read_attempt
+        elif isinstance(e, ConnectionError | httpx.ConnectError | httpx.ConnectTimeout):
+            error_type = "connect"
+            request.connect_attempt += 1
+            attempts = request.connect_attempt
+        else:
+            error_msg = f"Unexpected exception: {e!s}"
+            return [
+                ItemsFailedRequest2(
+                    ids=[item.as_id() for item in request.items],
+                    error_message=error_msg,
+                )
+            ]
+
+        if attempts <= self._max_retries:
+            time.sleep(self._backoff_time(request.total_attempts))
+            return [request]
+        else:
+            # We have already incremented the attempt count, so we subtract 1 here
+            error_msg = f"RequestException after {request.total_attempts - 1} attempts ({error_type} error): {e!s}"
+
+            return [
+                ItemsFailedRequest2(
+                    ids=[item.as_id() for item in request.items],
+                    error_message=error_msg,
+                )
+            ]

cognite_toolkit-0.7.30/cognite_toolkit/_cdf_tk/utils/http_client/_data_classes2.py

@@ -0,0 +1,187 @@
+import gzip
+import sys
+from abc import ABC, abstractmethod
+from collections.abc import Hashable
+from typing import Any, Literal
+
+import httpx
+from cognite.client import global_config
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, model_validator
+from pydantic.alias_generators import to_camel
+
+from cognite_toolkit._cdf_tk.utils.http_client._tracker import ItemsRequestTracker
+from cognite_toolkit._cdf_tk.utils.useful_types import PrimitiveType
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class HTTPResult2(BaseModel): ...
+
+
+class FailedRequest2(HTTPResult2):
+    error: str
+
+
+class SuccessResponse2(HTTPResult2):
+    status_code: int
+    body: str
+    content: bytes
+
+
+class ErrorDetails2(BaseModel):
+    """This is the expected structure of error details in the CDF API"""
+
+    code: int
+    message: str
+    missing: list[JsonValue] | None = None
+    duplicated: list[JsonValue] | None = None
+    is_auto_retryable: bool | None = None
+
+    @classmethod
+    def from_response(cls, response: httpx.Response) -> "ErrorDetails2":
+        """Populate the error details from a httpx response."""
+        try:
+            res = TypeAdapter(dict[Literal["error"], ErrorDetails2]).validate_json(response.text)
+        except ValueError:
+            return cls(code=response.status_code, message=response.text)
+        return res["error"]
+
+
+class FailedResponse2(HTTPResult2):
+    status_code: int
+    body: str
+    error: ErrorDetails2
+
+
+class BaseRequestMessage(BaseModel, ABC):
+    endpoint_url: str
+    method: Literal["GET", "POST", "PATCH", "DELETE", "PUT"]
+    connect_attempt: int = 0
+    read_attempt: int = 0
+    status_attempt: int = 0
+    api_version: str | None = None
+    content_type: str = "application/json"
+    accept: str = "application/json"
+
+    parameters: dict[str, PrimitiveType] | None = None
+
+    @property
+    def total_attempts(self) -> int:
+        return self.connect_attempt + self.read_attempt + self.status_attempt
+
+    @property
+    @abstractmethod
+    def content(self) -> str | bytes | None: ...
+
+
+class RequestMessage2(BaseRequestMessage):
+    data_content: bytes | None = None
+    body_content: dict[str, JsonValue] | None = None
+
+    @model_validator(mode="before")
+    def check_data_or_body(cls, values: dict[str, Any]) -> dict[str, Any]:
+        if values.get("data_content") is not None and values.get("body_content") is not None:
+            raise ValueError("Only one of data_content or body_content can be set.")
+        return values
+
+    @property
+    def content(self) -> str | bytes | None:
+        data: str | bytes | None = None
+        if self.data_content is not None:
+            data = self.data_content
+            if not global_config.disable_gzip:
+                data = gzip.compress(data)
+        elif self.body_content is not None:
+            # We serialize using pydantic instead of json.dumps. This is because pydantic is faster
+            # and handles more complex types such as datetime, float('nan'), etc.
+            data = _BODY_SERIALIZER.dump_json(self.body_content)
+            if not global_config.disable_gzip and isinstance(data, str):
+                data = gzip.compress(data.encode("utf-8"))
+        return data
+
+
+_BODY_SERIALIZER = TypeAdapter(dict[str, JsonValue])
+
+
+class ItemsResultMessage2(BaseModel):
+    ids: list[Hashable]
+
+
+class ItemsFailedRequest2(ItemsResultMessage2):
+    error_message: str
+
+
+class ItemsSuccessResponse2(ItemsResultMessage2):
+    status_code: int
+    body: str
+    content: bytes
+
+
+class ItemsFailedResponse2(ItemsResultMessage2):
+    status_code: int
+    error: ErrorDetails2
+    body: str
+
+
+class BaseModelObject(BaseModel):
+    """Base class for all object. This includes resources and nested objects."""
+
+    # We allow extra fields to support forward compatibility.
+    model_config = ConfigDict(alias_generator=to_camel, extra="allow")
+
+    def dump(self, camel_case: bool = True) -> dict[str, Any]:
+        """Dump the resource to a dictionary.
+
+        This is the default serialization method for request resources.
+        """
+        return self.model_dump(mode="json", by_alias=camel_case, exclude_unset=True)
+
+    @classmethod
+    def _load(cls, resource: dict[str, Any]) -> "Self":
+        """Load method to match CogniteResource signature."""
+        return cls.model_validate(resource)
+
+
+class RequestResource(BaseModelObject, ABC):
+    @abstractmethod
+    def as_id(self) -> Hashable: ...
+
+
+def _set_default_tracker(data: dict[str, Any]) -> ItemsRequestTracker:
+    if "tracker" not in data or data["tracker"] is None:
+        return ItemsRequestTracker(data.get("max_failures_before_abort", 50))
+    return data["tracker"]
+
+
+class ItemsRequest2(BaseRequestMessage):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    items: list[RequestResource]
+    extra_body_fields: dict[str, JsonValue] | None = None
+    max_failures_before_abort: int = 50
+    tracker: ItemsRequestTracker = Field(init=False, default_factory=_set_default_tracker, exclude=True)
+
+    @property
+    def content(self) -> str | bytes | None:
+        body: dict[str, JsonValue] = {"items": [item.dump() for item in self.items]}
+        if self.extra_body_fields:
+            body.update(self.extra_body_fields)
+        res = _BODY_SERIALIZER.dump_json(body)
+        if not global_config.disable_gzip and isinstance(res, str):
+            return gzip.compress(res.encode("utf-8"))
+        return res
+
+    def split(self, status_attempts: int) -> list["ItemsRequest2"]:
+        """Split the request into multiple requests with a single item each."""
+        mid = len(self.items) // 2
+        if mid == 0:
+            return [self]
+        self.tracker.register_failure()
+        messages: list[ItemsRequest2] = []
+        for part in (self.items[:mid], self.items[mid:]):
+            new_request = self.model_copy(update={"items": part, "status_attempt": status_attempts})
+            new_request.tracker = self.tracker
+            messages.append(new_request)
+        return messages