cognite-toolkit 0.7.30__py3-none-any.whl → 0.7.51__py3-none-any.whl

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py

@@ -4,20 +4,19 @@ from typing import Any, final
 from cognite.client.data_classes.capabilities import Capability, StreamsAcl
 from cognite.client.utils.useful_types import SequenceNotStr
 
-from cognite_toolkit._cdf_tk.client.data_classes.streams import (
+from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import ExternalId
+from cognite_toolkit._cdf_tk.client.resource_classes.streams import (
     StreamRequest,
     StreamResponse,
-    StreamResponseList,
 )
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
 from cognite_toolkit._cdf_tk.resource_classes import StreamYAML
-from cognite_toolkit._cdf_tk.utils.http_client import ToolkitAPIError
 
 from .datamodel import ContainerCRUD
 
 
 @final
-class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse]):
+class StreamCRUD(ResourceCRUD[ExternalId, StreamRequest, StreamResponse]):
     folder_name = "streams"
     resource_cls = StreamResponse
     resource_write_cls = StreamRequest
@@ -32,14 +31,14 @@ class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse]):
         return "streams"
 
     @classmethod
-    def get_id(cls, item: StreamRequest | StreamResponse | dict) -> str:
+    def get_id(cls, item: StreamRequest | StreamResponse | dict) -> ExternalId:
         if isinstance(item, dict):
-            return item["externalId"]
-        return item.external_id
+            return ExternalId(external_id=item["externalId"])
+        return ExternalId(external_id=item.external_id)
 
     @classmethod
-    def dump_id(cls, id: str) -> dict[str, Any]:
-        return {"externalId": id}
+    def dump_id(cls, id: ExternalId) -> dict[str, Any]:
+        return id.dump()
 
     @classmethod
     def get_required_capability(
@@ -55,30 +54,15 @@ class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse]):
         )
         return StreamsAcl(actions, StreamsAcl.Scope.All())
 
-    def create(self, items: Sequence[StreamRequest]) -> StreamResponseList:
-        created = self.client.streams.create(list(items))
-        return StreamResponseList(created)
+    def create(self, items: Sequence[StreamRequest]) -> list[StreamResponse]:
+        return self.client.streams.create(items)
 
-    def retrieve(self, ids: SequenceNotStr[str]) -> StreamResponseList:
-        retrieved: list[StreamResponse] = []
-        for _id in ids:
-            try:
-                _resp = self.client.streams.retrieve(_id)
-            except ToolkitAPIError:
-                continue
-            if _resp is not None:
-                retrieved.append(_resp)
-        return StreamResponseList(retrieved)
+    def retrieve(self, ids: SequenceNotStr[ExternalId]) -> list[StreamResponse]:
+        return self.client.streams.retrieve(list(ids), ignore_unknown_ids=True)
 
-    def delete(self, ids: SequenceNotStr[str]) -> int:
-        count = 0
-        for _id in ids:
-            try:
-                self.client.streams.delete(_id)
-            except ToolkitAPIError:
-                continue
-            count += 1
-        return count
+    def delete(self, ids: SequenceNotStr[ExternalId]) -> int:
+        self.client.streams.delete(list(ids), ignore_unknown_ids=True)
+        return len(ids)
 
     def _iterate(
         self,

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/timeseries.py

@@ -2,16 +2,13 @@ import json
 from collections.abc import Hashable, Iterable, Sequence
 from itertools import zip_longest
 from pathlib import Path
-from typing import Any, Literal, cast, final
+from typing import Any, Literal, final
 
 from cognite.client.data_classes import (
     DatapointSubscription,
     DatapointSubscriptionList,
     DataPointSubscriptionUpdate,
     DataPointSubscriptionWrite,
-    TimeSeries,
-    TimeSeriesList,
-    TimeSeriesWrite,
 )
 from cognite.client.data_classes.capabilities import (
     Capability,
@@ -23,6 +20,9 @@ from cognite.client.data_classes.datapoints_subscriptions import TimeSeriesIDLis
 from cognite.client.exceptions import CogniteAPIError, CogniteNotFoundError
 from cognite.client.utils.useful_types import SequenceNotStr
 
+from cognite_toolkit._cdf_tk.client.request_classes.filters import ClassicFilter
+from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import ExternalId, InternalOrExternalId
+from cognite_toolkit._cdf_tk.client.resource_classes.timeseries import TimeSeriesRequest, TimeSeriesResponse
 from cognite_toolkit._cdf_tk.constants import MAX_TIMESTAMP_MS, MIN_TIMESTAMP_MS
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceContainerCRUD, ResourceCRUD
 from cognite_toolkit._cdf_tk.exceptions import (
@@ -41,11 +41,11 @@ from .data_organization import DataSetsCRUD
 
 
 @final
-class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries]):
+class TimeSeriesCRUD(ResourceContainerCRUD[ExternalId, TimeSeriesRequest, TimeSeriesResponse]):
     item_name = "datapoints"
     folder_name = "timeseries"
-    resource_cls = TimeSeries
-    resource_write_cls = TimeSeriesWrite
+    resource_cls = TimeSeriesResponse
+    resource_write_cls = TimeSeriesRequest
     yaml_cls = TimeSeriesYAML
     kind = "TimeSeries"
     dependencies = frozenset({DataSetsCRUD, GroupAllScopedCRUD, AssetCRUD})
@@ -57,7 +57,7 @@ class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries]):
 
     @classmethod
     def get_required_capability(
-        cls, items: Sequence[TimeSeriesWrite] | None, read_only: bool
+        cls, items: Sequence[TimeSeriesRequest] | None, read_only: bool
     ) -> Capability | list[Capability]:
         if not items and items is not None:
             return []
@@ -72,22 +72,22 @@ class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries]):
         return TimeSeriesAcl(actions, scope)
 
     @classmethod
-    def get_id(cls, item: TimeSeries | TimeSeriesWrite | dict) -> str:
+    def get_id(cls, item: TimeSeriesRequest | TimeSeriesResponse | dict) -> ExternalId:
         if isinstance(item, dict):
-            return item["externalId"]
+            return ExternalId(external_id=item["externalId"])
         if item.external_id is None:
             raise ToolkitRequiredValueError("TimeSeries must have external_id set.")
-        return item.external_id
+        return ExternalId(external_id=item.external_id)
 
     @classmethod
-    def get_internal_id(cls, item: TimeSeries | dict) -> int:
+    def get_internal_id(cls, item: TimeSeriesResponse | dict) -> int:
         if isinstance(item, dict):
             return item["id"]
         return item.id
 
     @classmethod
-    def dump_id(cls, id: str) -> dict[str, Any]:
-        return {"externalId": id}
+    def dump_id(cls, id: ExternalId) -> dict[str, Any]:
+        return id.dump()
 
     @classmethod
     def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
@@ -97,9 +97,9 @@ class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries]):
             for security_category in item["securityCategoryNames"]:
                 yield SecurityCategoryCRUD, security_category
         if "assetExternalId" in item:
-            yield AssetCRUD, item["assetExternalId"]
+            yield AssetCRUD, ExternalId(external_id=item["assetExternalId"])
 
-    def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> TimeSeriesWrite:
+    def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> TimeSeriesRequest:
         if ds_external_id := resource.pop("dataSetExternalId", None):
             resource["dataSetId"] = self.client.lookup.data_sets.id(ds_external_id, is_dry_run)
         if security_categories_names := resource.pop("securityCategoryNames", []):
@@ -108,10 +108,10 @@ class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries]):
             )
         if asset_external_id := resource.pop("assetExternalId", None):
             resource["assetId"] = self.client.lookup.assets.id(asset_external_id, is_dry_run)
-        return TimeSeriesWrite._load(resource)
+        return TimeSeriesRequest.model_validate(resource)
 
-    def dump_resource(self, resource: TimeSeries, local: dict[str, Any] | None = None) -> dict[str, Any]:
-        dumped = resource.as_write().dump()
+    def dump_resource(self, resource: TimeSeriesResponse, local: dict[str, Any] | None = None) -> dict[str, Any]:
+        dumped = resource.as_request_resource().dump()
         if data_set_id := dumped.pop("dataSetId", None):
             dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id)
         if security_categories := dumped.pop("securityCategories", []):
@@ -120,53 +120,48 @@ class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries]):
             dumped["assetExternalId"] = self.client.lookup.assets.external_id(asset_id)
         return dumped
 
-    def create(self, items: Sequence[TimeSeriesWrite]) -> TimeSeriesList:
-        return self.client.time_series.create(items)
+    def create(self, items: Sequence[TimeSeriesRequest]) -> list[TimeSeriesResponse]:
+        return self.client.tool.timeseries.create(items)
 
-    def retrieve(self, ids: SequenceNotStr[str | int]) -> TimeSeriesList:
-        internal_ids, external_ids = self._split_ids(ids)
-        return self.client.time_series.retrieve_multiple(
-            ids=internal_ids, external_ids=external_ids, ignore_unknown_ids=True
-        )
+    def retrieve(self, ids: SequenceNotStr[ExternalId]) -> list[TimeSeriesResponse]:
+        return self.client.tool.timeseries.retrieve(list(ids), ignore_unknown_ids=True)
 
-    def update(self, items: Sequence[TimeSeriesWrite]) -> TimeSeriesList:
-        return self.client.time_series.update(items, mode="replace")
+    def update(self, items: Sequence[TimeSeriesRequest]) -> list[TimeSeriesResponse]:
+        return self.client.tool.timeseries.update(items, mode="replace")
 
-    def delete(self, ids: SequenceNotStr[str | int]) -> int:
-        existing = self.retrieve(ids)
-        if existing:
-            self.client.time_series.delete(id=existing.as_ids(), ignore_unknown_ids=True)
-        return len(existing)
+    def delete(self, ids: SequenceNotStr[InternalOrExternalId]) -> int:
+        if not ids:
+            return 0
+        self.client.tool.timeseries.delete(list(ids), ignore_unknown_ids=True)
+        return len(ids)
 
     def _iterate(
         self,
         data_set_external_id: str | None = None,
         space: str | None = None,
         parent_ids: list[Hashable] | None = None,
-    ) -> Iterable[TimeSeries]:
-        return iter(
-            self.client.time_series(data_set_external_ids=[data_set_external_id] if data_set_external_id else None)
-        )
+    ) -> Iterable[TimeSeriesResponse]:
+        filter_ = ClassicFilter.from_asset_subtree_and_data_sets(data_set_id=data_set_external_id)
+        for timeseries in self.client.tool.timeseries.iterate(filter=filter_, limit=None):
+            yield from timeseries
 
-    def count(self, ids: str | dict[str, Any] | SequenceNotStr[str | dict[str, Any]] | None) -> int:
+    def count(self, ids: SequenceNotStr[ExternalId]) -> int:
         datapoints = self.client.time_series.data.retrieve(
-            external_id=ids,  # type: ignore[arg-type]
+            external_id=[id.external_id for id in ids],
             start=MIN_TIMESTAMP_MS,
             end=MAX_TIMESTAMP_MS + 1,
             aggregates="count",
             granularity="1000d",
             ignore_unknown_ids=True,
         )
-        return sum(sum(data.count or []) for data in datapoints)  # type: ignore[union-attr, misc, arg-type]
+        return sum(sum(data.count or []) for data in datapoints)
 
-    def drop_data(self, ids: SequenceNotStr[str] | None) -> int:
+    def drop_data(self, ids: SequenceNotStr[ExternalId]) -> int:
         count = self.count(ids)
-        existing = self.client.time_series.retrieve_multiple(
-            external_ids=cast(SequenceNotStr[str], ids), ignore_unknown_ids=True
-        ).as_external_ids()
-        for external_id in existing:
+        existing = self.client.tool.timeseries.retrieve(list(ids), ignore_unknown_ids=True)
+        for ts in existing:
             self.client.time_series.data.delete_range(
-                external_id=external_id, start=MIN_TIMESTAMP_MS, end=MAX_TIMESTAMP_MS + 1
+                external_id=ts.external_id, start=MIN_TIMESTAMP_MS, end=MAX_TIMESTAMP_MS + 1
             )
         return count
 
@@ -219,7 +214,7 @@ class DatapointSubscriptionCRUD(
         if "dataSetExternalId" in item:
             yield DataSetsCRUD, item["dataSetExternalId"]
         for timeseries_id in item.get("timeSeriesIds", []):
-            yield TimeSeriesCRUD, timeseries_id
+            yield TimeSeriesCRUD, ExternalId(external_id=timeseries_id)
 
     @classmethod
     def get_required_capability(

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py

@@ -26,6 +26,8 @@
 # limitations under the License.
 
 
+import random
+import time
 import warnings
 from collections import defaultdict
 from collections.abc import Callable, Hashable, Iterable, Sequence
@@ -63,7 +65,7 @@ from rich import print
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.raw import RawDatabase, RawTable
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.raw import RawDatabase, RawTable
 from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
 from cognite_toolkit._cdf_tk.exceptions import (
@@ -427,25 +429,58 @@ class TransformationCRUD(ResourceCRUD[str, TransformationWrite, Transformation])
                     raise error from e
                 raise e
             except CogniteAPIError as e:
-                if "Failed to bind session using nonce for" in e.message and len(chunk) > 1:
-                    MediumSeverityWarning(
-                        f"Failed to create {len(chunk)} transformations in a batch due to nonce binding error. "
-                        "Trying to recover by creating them one by one."
-                    ).print_warning(console=self.console)
-                    # Retry one by one
-                    for item in chunk:
-                        recovered = self._execute_in_batches(items=[item], api_call=api_call)
-                        results.extend(recovered)
-                    if self.console:
-                        self.console.print(
-                            f"  [bold green]RECOVERED:[/] Successfully created {len(chunk)} transformations one by one."
-                        )
+                if "Failed to bind session using nonce" in e.message and len(chunk) > 1:
+                    results.extend(self._execute_one_by_one(chunk, api_call))
                 else:
                     raise
             else:
                 results.extend(chunk_results)
         return results
 
+    def _execute_one_by_one(
+        self,
+        chunk: Sequence[TransformationWrite],
+        api_call: Callable[[Sequence[TransformationWrite]], TransformationList],
+    ) -> TransformationList:
+        MediumSeverityWarning(
+            f"Failed to create {len(chunk)} transformations in a batch due to nonce binding error. "
+            "Trying to recover by creating them one by one."
+        ).print_warning(console=self.client.console)
+        # Retry one by one
+        failed_ids: list[str] = []
+        success_count = 0
+        delay = 0.3
+        self._sleep_with_jitter(delay, delay + 0.3)
+        results = TransformationList([])
+        for item in chunk:
+            try:
+                recovered = api_call([item])
+            except CogniteAPIError as e:
+                if "Failed to bind session using nonce" in e.message:
+                    failed_ids.append(item.external_id or "<missing>")
+                    self._sleep_with_jitter(delay, delay + 0.3)
+                else:
+                    raise
+            else:
+                results.extend(recovered)
+                success_count += 1
+        message = f"  [bold]RECOVERY COMPLETE:[/] Successfully created {success_count:,} transformations"
+        if failed_ids:
+            message += f", failed to create {len(failed_ids):,} transformations: {humanize_collection(failed_ids)}"
+        else:
+            message += "."
+        if failed_ids:
+            HighSeverityWarning(message).print_warning(include_timestamp=True, console=self.client.console)
+        else:
+            self.client.console.print(message)
+        return results
+
+    @staticmethod
+    def _sleep_with_jitter(base_delay: float, max_delay: float) -> None:
+        """Sleeps for a random duration between base_delay and max_delay (inclusive)."""
+        sleep_time = random.uniform(base_delay, max_delay)
+        time.sleep(sleep_time)
+
     def _update_nonce(self, items: Sequence[TransformationWrite]) -> None:
         for item in items:
             if not item.external_id:

cognite_toolkit/_cdf_tk/data_classes/__init__.py

@@ -29,6 +29,7 @@ from ._deploy_results import (
     ResourceDeployResult,
     UploadDeployResult,
 )
+from ._issues import Issue, IssueList
 from ._module_directories import ModuleDirectories, ModuleLocation
 from ._module_resources import ModuleResources
 from ._packages import Package, Packages
@@ -56,6 +57,8 @@ __all__ = [
     "DeployResults",
     "Environment",
     "InitConfigYAML",
+    "Issue",
+    "IssueList",
     "ModuleDirectories",
     "ModuleLocation",
     "ModuleResources",

cognite_toolkit/_cdf_tk/data_classes/_issues.py

@@ -0,0 +1,36 @@
+import sys
+from collections import UserList
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from pydantic import BaseModel, Field
+
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
+
+MODULE_ISSUE_CODE = "MOD"
+
+
+class Issue(BaseModel):
+    """Base class for all issues"""
+
+    code: str
+    message: str | None = Field(default=None)
+
+
+# temporary adapter to manage existing warnings
+class IssueList(UserList[Issue]):
+    """List of build issues."""
+
+    @classmethod
+    def from_warning_list(cls, warning_list: WarningList[ToolkitWarning]) -> Self:
+        """Create a IssueList from a WarningList."""
+        return cls([Issue(code="WARN", message=warning.get_message()) for warning in warning_list])
+
+
+class ModuleLoadingIssue(Issue):
+    """Issue with the loading of a module folder."""
+
+    code: str = "MOD_001"

cognite_toolkit/_cdf_tk/data_classes/_module_directories.py

@@ -8,7 +8,8 @@ from pathlib import Path
 from typing import Any, SupportsIndex, overload
 
 from cognite_toolkit._cdf_tk.constants import INDEX_PATTERN
-from cognite_toolkit._cdf_tk.utils import calculate_directory_hash, iterate_modules, resource_folder_from_path
+from cognite_toolkit._cdf_tk.utils.hashing import calculate_directory_hash
+from cognite_toolkit._cdf_tk.utils.modules import iterate_modules, resource_folder_from_path
 
 from ._module_toml import ModuleToml
 

cognite_toolkit/_cdf_tk/data_classes/_module_toml.py

@@ -68,6 +68,7 @@ class ModuleToml:
         title: str | None = None
         id: str | None = None
         is_selected_by_default: bool = False
+        package_id: str | None = None
         if "module" in data:
             title = data["module"].get("title")
             id = data["module"].get("id")

cognite_toolkit/_cdf_tk/data_classes/_tracking_info.py

@@ -32,6 +32,10 @@ class CommandTrackingInfo(BaseModel):
     downloaded_library_ids: set[str] = Field(default_factory=set, alias="downloadedLibraryIds")
     downloaded_package_ids: set[str] = Field(default_factory=set, alias="downloadedPackageIds")
     downloaded_module_ids: set[str] = Field(default_factory=set, alias="downloadedModuleIds")
+    function_validation_count: int = Field(default=0, alias="functionValidationCount")
+    function_validation_failures: int = Field(default=0, alias="functionValidationFailures")
+    function_validation_credential_errors: int = Field(default=0, alias="functionValidationCredentialErrors")
+    function_validation_time_ms: int = Field(default=0, alias="functionValidationTimeMs")
 
     def to_dict(self) -> dict[str, Any]:
         """Convert the tracking info to a dictionary for Mixpanel.

cognite_toolkit/_cdf_tk/feature_flags.py

@@ -61,6 +61,14 @@ class Flags(Enum):
         visible=True,
         description="Enables extended upload to support uploading individual files",
     )
+    SIMULATORS = FlagMetadata(
+        visible=True,
+        description="Enables the support for simulator model resources",
+    )
+    FUNCTION_REQUIREMENTS_VALIDATION = FlagMetadata(
+        visible=True,
+        description="Enables validation of function requirements.txt during build using pip dry-run",
+    )
 
     def is_enabled(self) -> bool:
         return FeatureFlag.is_enabled(self)

cognite_toolkit/_cdf_tk/resource_classes/__init__.py

@@ -40,6 +40,7 @@ from .robotics import RobotCapabilityYAML, RobotDataPostProcessingYAML, RobotFra
 from .search_config import SearchConfigYAML
 from .securitycategories import SecurityCategoriesYAML
 from .sequence import SequenceRowYAML, SequenceYAML
+from .simulator_model import SimulatorModelYAML
 from .space import SpaceYAML
 from .streamlit_ import StreamlitYAML
 from .streams import StreamYAML
@@ -94,6 +95,7 @@ __all__ = [
     "SecurityCategoriesYAML",
     "SequenceRowYAML",
     "SequenceYAML",
+    "SimulatorModelYAML",
     "SpaceYAML",
     "StreamYAML",
     "StreamlitYAML",

cognite_toolkit/_cdf_tk/resource_classes/capabilities.py

@@ -510,6 +510,12 @@ class LegacyGenericsAcl(Capability):
     scope: AllScope
 
 
+class SimulatorsAcl(Capability):
+    _capability_name = "simulatorsAcl"
+    actions: list[Literal["READ", "WRITE", "DELETE", "RUN", "MANAGE"]]
+    scope: AllScope | DataSetScope
+
+
 class StreamsAcl(Capability):
     _capability_name = "streamsAcl"
     actions: list[Literal["READ", "CREATE", "DELETE"]]

cognite_toolkit/_cdf_tk/resource_classes/search_config.py

@@ -2,7 +2,7 @@ from typing import ClassVar
 
 from pydantic import Field
 
-from cognite_toolkit._cdf_tk.client.data_classes.search_config import SearchConfigWrite
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.search_config import SearchConfigWrite
 
 from .base import BaseModelResource, ToolkitResource
 

cognite_toolkit/_cdf_tk/resource_classes/simulator_model.py

@@ -0,0 +1,17 @@
+from pydantic import Field
+
+from .base import ToolkitResource
+
+
+class SimulatorModelYAML(ToolkitResource):
+    """Simulator model YAML resource class.
+
+    Based on: https://api-docs.cognite.com/20230101/tag/Simulator-Models/operation/create_simulator_model_simulators_models_post
+    """
+
+    external_id: str = Field(description="External ID of the simulator model.", min_length=1, max_length=255)
+    simulator_external_id: str = Field(description="External id of the simulator.", min_length=1, max_length=50)
+    name: str = Field(description="The name of the simulator model.", min_length=1, max_length=50)
+    description: str | None = Field(None, description="Description of the simulator model.", max_length=500)
+    data_set_external_id: str = Field(description="The external ID of the dataset this simulator model belongs to.")
+    type: str = Field(description="The type of the simulator model.", min_length=1, max_length=50)