cognite-toolkit 0.7.30__py3-none-any.whl → 0.7.39__py3-none-any.whl

cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py

@@ -38,9 +38,12 @@ from ._instances import (
     SelectedView,
 )
 from ._raw import RawTableSelector, SelectedTable
+from ._three_d import ThreeDModelFilteredSelector, ThreeDModelIdSelector, ThreeDSelector
 
 Selector = Annotated[
     RawTableSelector
+    | ThreeDModelIdSelector
+    | ThreeDModelFilteredSelector
     | InstanceViewSelector
     | InstanceFileSelector
     | InstanceSpaceSelector
@@ -60,7 +63,7 @@ Selector = Annotated[
 ]
 
 ALPHA_SELECTORS = {FileIdentifierSelector}
-
+INTERNAL = {ThreeDModelIdSelector, ThreeDModelFilteredSelector}
 SelectorAdapter: TypeAdapter[Selector] = TypeAdapter(Selector)
 
 
@@ -83,6 +86,10 @@ def load_selector(manifest_file: Path) -> Selector | ToolkitWarning:
         return MediumSeverityWarning(
             f"Selector type '{type(selector).__name__}' in file '{manifest_file}' is in alpha. To enable it set the alpha flag 'extend-upload = true' in your CDF.toml file."
         )
+    elif type(selector) in INTERNAL:
+        return MediumSeverityWarning(
+            f"Selector type '{type(selector).__name__}' in file '{manifest_file}' is for internal use only and cannot be used."
+        )
     return selector
 
 
@@ -119,6 +126,8 @@ __all__ = [
     "SelectedView",
     "Selector",
     "SelectorAdapter",
+    "ThreeDModelIdSelector",
+    "ThreeDSelector",
     "TimeSeriesColumn",
     "load_selector",
 ]

cognite_toolkit/_cdf_tk/storageio/selectors/_three_d.py

@@ -0,0 +1,34 @@
+import hashlib
+from abc import ABC
+from typing import Literal
+
+from ._base import DataSelector
+
+
+class ThreeDSelector(DataSelector, ABC):
+    kind: Literal["3D"] = "3D"
+
+    @property
+    def group(self) -> str:
+        return "3DModels"
+
+
+class ThreeDModelFilteredSelector(ThreeDSelector):
+    type: Literal["3DFiltered"] = "3DFiltered"
+    model_type: Literal["Classic", "DataModel"] = "Classic"
+    published: bool | None = None
+
+    def __str__(self) -> str:
+        suffix = f"3DModels_{self.model_type}"
+        if self.published is not None:
+            return f"{suffix}_published_{self.published}"
+        return suffix
+
+
+class ThreeDModelIdSelector(ThreeDSelector):
+    type: Literal["3DId"] = "3DId"
+    ids: tuple[int, ...]
+
+    def __str__(self) -> str:
+        hash_ = hashlib.md5(",".join(sorted(map(str, self.ids))).encode()).hexdigest()[:8]
+        return f"3DModels_ids_count_{len(self.ids)}_hash_{hash_}"

cognite_toolkit/_cdf_tk/utils/cdf.py

@@ -21,7 +21,7 @@ from filelock import BaseFileLock, FileLock, Timeout
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient, ToolkitClientConfig
-from cognite_toolkit._cdf_tk.client.data_classes.raw import RawTable
+from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawTable
 from cognite_toolkit._cdf_tk.constants import ENV_VAR_PATTERN, MAX_ROW_ITERATION_RUN_QUERY, MAX_RUN_QUERY_FREQUENCY_MIN
 from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitError,

cognite_toolkit/_cdf_tk/utils/http_client/__init__.py

@@ -17,25 +17,53 @@ from ._data_classes import (
     SuccessResponse,
     SuccessResponseItems,
 )
+from ._data_classes2 import (
+    BaseModelObject,
+    ErrorDetails2,
+    FailedRequest2,
+    FailedResponse2,
+    HTTPResult2,
+    ItemsFailedRequest2,
+    ItemsFailedResponse2,
+    ItemsRequest2,
+    ItemsResultMessage2,
+    ItemsSuccessResponse2,
+    RequestMessage2,
+    RequestResource,
+    SuccessResponse2,
+)
 from ._exception import ToolkitAPIError
 
 __all__ = [
+    "BaseModelObject",
     "DataBodyRequest",
     "ErrorDetails",
+    "ErrorDetails2",
+    "FailedRequest2",
     "FailedRequestItems",
     "FailedRequestMessage",
     "FailedResponse",
+    "FailedResponse2",
     "FailedResponseItems",
     "HTTPClient",
     "HTTPMessage",
+    "HTTPResult2",
     "ItemMessage",
+    "ItemsFailedRequest2",
+    "ItemsFailedResponse2",
     "ItemsRequest",
+    "ItemsRequest2",
+    "ItemsResultMessage2",
+    "ItemsSuccessResponse2",
     "ParamRequest",
     "RequestMessage",
+    "RequestMessage2",
+    "RequestResource",
     "ResponseList",
     "ResponseMessage",
     "SimpleBodyRequest",
     "SuccessResponse",
+    "SuccessResponse2",
     "SuccessResponseItems",
     "ToolkitAPIError",
 ]

cognite_toolkit/_cdf_tk/utils/http_client/_client.py

@@ -32,6 +32,7 @@ from cognite_toolkit._cdf_tk.utils.http_client._data_classes2 import (
     ItemsFailedRequest2,
     ItemsFailedResponse2,
     ItemsRequest2,
+    ItemsResultList,
     ItemsResultMessage2,
     ItemsSuccessResponse2,
     RequestMessage2,
@@ -424,7 +425,7 @@ class HTTPClient:
             results = self._handle_items_error(e, message)
         return results
 
-    def request_items_retries(self, message: ItemsRequest2) -> Sequence[ItemsResultMessage2]:
+    def request_items_retries(self, message: ItemsRequest2) -> ItemsResultList:
         """Send an HTTP request with multiple items and handle retries.
 
         This method will keep retrying the request until it either succeeds or
@@ -442,7 +443,7 @@ class HTTPClient:
             raise RuntimeError(f"ItemsRequest2 has already been attempted {message.total_attempts} times.")
         pending_requests: deque[ItemsRequest2] = deque()
         pending_requests.append(message)
-        final_responses: list[ItemsResultMessage2] = []
+        final_responses = ItemsResultList([])
         while pending_requests:
             current_request = pending_requests.popleft()
             results = self.request_items(current_request)

cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py

@@ -76,6 +76,9 @@ class ErrorDetails:
 class FailedRequestMessage(HTTPMessage):
     error: str
 
+    def __str__(self) -> str:
+        return self.error
+
 
 @dataclass
 class ResponseMessage(HTTPMessage):
@@ -135,6 +138,9 @@ class FailedResponse(ResponseMessage):
         output["error"] = self.error.dump()
         return output
 
+    def __str__(self) -> str:
+        return f"{self.error.code} | {self.error.message}"
+
 
 @dataclass
 class SimpleRequest(RequestMessage):

cognite_toolkit/_cdf_tk/utils/http_client/_data_classes2.py

@@ -1,7 +1,8 @@
 import gzip
 import sys
 from abc import ABC, abstractmethod
-from collections.abc import Hashable
+from collections import UserList
+from collections.abc import Hashable, Sequence
 from typing import Any, Literal
 
 import httpx
@@ -9,6 +10,7 @@ from cognite.client import global_config
 from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, model_validator
 from pydantic.alias_generators import to_camel
 
+from cognite_toolkit._cdf_tk.utils.http_client._exception import ToolkitAPIError
 from cognite_toolkit._cdf_tk.utils.http_client._tracker import ItemsRequestTracker
 from cognite_toolkit._cdf_tk.utils.useful_types import PrimitiveType
 
@@ -18,7 +20,17 @@ else:
     from typing_extensions import Self
 
 
-class HTTPResult2(BaseModel): ...
+class HTTPResult2(BaseModel):
+    def get_success_or_raise(self) -> "SuccessResponse2":
+        """Raises an exception if any response in the list indicates a failure."""
+        if isinstance(self, SuccessResponse2):
+            return self
+        elif isinstance(self, FailedResponse2):
+            raise ToolkitAPIError(f"Request failed with status code {self.status_code}: {self.error.message}")
+        elif isinstance(self, FailedRequest2):
+            raise ToolkitAPIError(f"Request failed with error: {self.error}")
+        else:
+            raise ToolkitAPIError("Unknown HTTPResult2 type")
 
 
 class FailedRequest2(HTTPResult2):
@@ -30,6 +42,11 @@ class SuccessResponse2(HTTPResult2):
     body: str
     content: bytes
 
+    @property
+    def body_json(self) -> dict[str, Any]:
+        """Parse the response body as JSON."""
+        return TypeAdapter(dict[str, JsonValue]).validate_json(self.body)
+
 
 class ErrorDetails2(BaseModel):
     """This is the expected structure of error details in the CDF API"""
@@ -98,8 +115,8 @@ class RequestMessage2(BaseRequestMessage):
             # We serialize using pydantic instead of json.dumps. This is because pydantic is faster
             # and handles more complex types such as datetime, float('nan'), etc.
             data = _BODY_SERIALIZER.dump_json(self.body_content)
-            if not global_config.disable_gzip and isinstance(data, str):
-                data = gzip.compress(data.encode("utf-8"))
+            if not global_config.disable_gzip and isinstance(data, bytes):
+                data = gzip.compress(data)
         return data
 
 
@@ -158,7 +175,7 @@ def _set_default_tracker(data: dict[str, Any]) -> ItemsRequestTracker:
 
 class ItemsRequest2(BaseRequestMessage):
     model_config = ConfigDict(arbitrary_types_allowed=True)
-    items: list[RequestResource]
+    items: Sequence[RequestResource]
     extra_body_fields: dict[str, JsonValue] | None = None
     max_failures_before_abort: int = 50
     tracker: ItemsRequestTracker = Field(init=False, default_factory=_set_default_tracker, exclude=True)
@@ -169,8 +186,8 @@ class ItemsRequest2(BaseRequestMessage):
         if self.extra_body_fields:
             body.update(self.extra_body_fields)
         res = _BODY_SERIALIZER.dump_json(body)
-        if not global_config.disable_gzip and isinstance(res, str):
-            return gzip.compress(res.encode("utf-8"))
+        if not global_config.disable_gzip and isinstance(res, bytes):
+            return gzip.compress(res)
         return res
 
     def split(self, status_attempts: int) -> list["ItemsRequest2"]:
@@ -185,3 +202,46 @@ class ItemsRequest2(BaseRequestMessage):
             new_request.tracker = self.tracker
             messages.append(new_request)
         return messages
+
+
+class ItemResponse(BaseModel):
+    items: list[dict[str, JsonValue]]
+
+
+class ItemsResultList(UserList[ItemsResultMessage2]):
+    def __init__(self, collection: Sequence[ItemsResultMessage2] | None = None) -> None:
+        super().__init__(collection or [])
+
+    def raise_for_status(self) -> None:
+        """Raises an exception if any response in the list indicates a failure."""
+        failed_responses = [resp for resp in self.data if isinstance(resp, ItemsFailedResponse2)]
+        failed_requests = [resp for resp in self.data if isinstance(resp, ItemsFailedRequest2)]
+        if not failed_responses and not failed_requests:
+            return
+        error_messages = "; ".join(f"Status {err.status_code}: {err.error.message}" for err in failed_responses)
+        if failed_requests:
+            if error_messages:
+                error_messages += "; "
+            error_messages += "; ".join(f"Request error: {err.error_message}" for err in failed_requests)
+        raise ToolkitAPIError(f"One or more requests failed: {error_messages}")
+
+    @property
+    def has_failed(self) -> bool:
+        """Indicates whether any response in the list indicates a failure.
+
+        Returns:
+            bool: True if there are any failed responses or requests, False otherwise.
+        """
+        for resp in self.data:
+            if isinstance(resp, ItemsFailedResponse2 | ItemsFailedRequest2):
+                return True
+        return False
+
+    def get_items(self) -> list[dict[str, JsonValue]]:
+        """Get the items from all successful responses."""
+        items: list[dict[str, JsonValue]] = []
+        for resp in self.data:
+            if isinstance(resp, ItemsSuccessResponse2):
+                body_json = ItemResponse.model_validate_json(resp.body)
+                items.extend(body_json.items)
+        return items

cognite_toolkit/_cdf_tk/utils/http_client/_tracker.py

@@ -1,5 +1,6 @@
 import threading
 from dataclasses import dataclass, field
+from typing import Any
 
 
 @dataclass
@@ -9,13 +10,15 @@ class ItemsRequestTracker:
     Attributes:
         max_failures_before_abort (int): Maximum number of allowed failed split requests before aborting
             the entire operation. A value of -1 indicates no early abort.
-        lock (threading.Lock): A lock to ensure thread-safe updates to the failure count.
+        lock (Any): A lock to ensure thread-safe updates to the failure count.
         failed_split_count (int): The current count of failed split requests.
 
     """
 
     max_failures_before_abort: int = -1  # -1 means no early abort
-    lock: threading.Lock = field(default_factory=threading.Lock, init=False)
+    # NOTE: `threading.Lock` is a factory function (backed by `_thread.allocate_lock`), not a type.
+    # Annotate as `Any` so Pydantic won't attempt to generate a schema for the lock field.
+    lock: Any = field(default_factory=threading.Lock, init=False)
     failed_split_count: int = field(default=0, init=False)
 
     def register_failure(self) -> None:

cognite_toolkit/_cdf_tk/utils/interactive_select.py

@@ -21,10 +21,11 @@ from questionary import Choice
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.canvas import Canvas
-from cognite_toolkit._cdf_tk.client.data_classes.charts import Chart, ChartList, Visibility
-from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMapping
-from cognite_toolkit._cdf_tk.client.data_classes.raw import RawTable
+from cognite_toolkit._cdf_tk.client.data_classes.legacy.canvas import Canvas
+from cognite_toolkit._cdf_tk.client.data_classes.legacy.charts import Chart, ChartList, Visibility
+from cognite_toolkit._cdf_tk.client.data_classes.legacy.migration import ResourceViewMapping
+from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawTable
+from cognite_toolkit._cdf_tk.client.data_classes.three_d import ThreeDModelResponse
 from cognite_toolkit._cdf_tk.exceptions import ToolkitMissingResourceError, ToolkitValueError
 
 from . import humanize_collection
@@ -823,3 +824,49 @@ class ResourceViewMappingInteractiveSelect:
                 f"Selected Resource View Mapping is not a valid ResourceViewMapping object: {selected_mapping!r}"
             )
         return selected_mapping
+
+
+class ThreeDInteractiveSelect:
+    def __init__(self, client: ToolkitClient, operation: str) -> None:
+        self.client = client
+        self.operation = operation
+
+    def select_three_d_models(self, model_type: Literal["classic", "dm"] | None = None) -> list[ThreeDModelResponse]:
+        """Select multiple 3D models interactively."""
+        if model_type is None:
+            model_type = questionary.select(
+                f"What type of 3D models do you want to {self.operation}?",
+                choices=[
+                    Choice(title="Classic models", value="classic"),
+                    Choice(title="Data modeling 3D", value="dm"),
+                ],
+            ).ask()
+        if model_type is None:
+            raise ToolkitValueError("No 3D model type selected.")
+        published = questionary.select(
+            f"Do you want to {self.operation} published or unpublished 3D models?",
+            choices=[
+                Choice(title="Published models", value=True),
+                Choice(title="Unpublished models", value=False),
+                Choice(title="Both published and unpublished models", value=None),
+            ],
+        ).ask()
+
+        models = self.client.tool.three_d.models.list(published=published, include_revision_info=True, limit=None)
+        if model_type == "classic":
+            models = [model for model in models if model.space is None]
+        else:
+            models = [model for model in models if model.space is not None]
+        if not models:
+            raise ToolkitMissingResourceError(
+                f"No 3D models found for type {model_type!r} with published={published!r}."
+            )
+
+        choices = [Choice(title=f"{model.name} ({model.id})", value=model) for model in models]
+        selected_models = questionary.checkbox(
+            f"Select 3D models to {self.operation}:",
+            choices=choices,
+        ).ask()
+        if selected_models is None or len(selected_models) == 0:
+            raise ToolkitValueError("No 3D models selected.")
+        return selected_models

cognite_toolkit/_cdf_tk/validation.py

@@ -182,6 +182,10 @@ def humanize_validation_error(error: ValidationError) -> list[str]:
             "dict_type",
         }:
             msg = f"{item['msg']}. Got {item['input']!r} of type {type(item['input']).__name__}."
+        elif error_type == "union_tag_not_found" and "ctx" in item and "discriminator" in item["ctx"]:
+            # This is when we use a discriminator field to determine the type in a union. For the user, this means they
+            # are missing a required field.
+            msg = f"Missing required field: {item['ctx']['discriminator']}"
         else:
             # Default to the Pydantic error message
             msg = item["msg"]

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.7.30
+      image: cognite/toolkit:0.7.39
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.7.30
+      image: cognite/toolkit:0.7.39
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.30"
+version = "0.7.39"
 
 
 [plugins]

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.7.30"
+__version__ = "0.7.39"

{cognite_toolkit-0.7.30.dist-info → cognite_toolkit-0.7.39.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.30
+Version: 0.7.39
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>