cognite-toolkit 0.7.39__py3-none-any.whl → 0.7.51__py3-none-any.whl

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/timeseries.py

@@ -2,16 +2,13 @@ import json
 from collections.abc import Hashable, Iterable, Sequence
 from itertools import zip_longest
 from pathlib import Path
-from typing import Any, Literal, cast, final
+from typing import Any, Literal, final
 
 from cognite.client.data_classes import (
     DatapointSubscription,
     DatapointSubscriptionList,
     DataPointSubscriptionUpdate,
     DataPointSubscriptionWrite,
-    TimeSeries,
-    TimeSeriesList,
-    TimeSeriesWrite,
 )
 from cognite.client.data_classes.capabilities import (
     Capability,
@@ -23,6 +20,9 @@ from cognite.client.data_classes.datapoints_subscriptions import TimeSeriesIDLis
 from cognite.client.exceptions import CogniteAPIError, CogniteNotFoundError
 from cognite.client.utils.useful_types import SequenceNotStr
 
+from cognite_toolkit._cdf_tk.client.request_classes.filters import ClassicFilter
+from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import ExternalId, InternalOrExternalId
+from cognite_toolkit._cdf_tk.client.resource_classes.timeseries import TimeSeriesRequest, TimeSeriesResponse
 from cognite_toolkit._cdf_tk.constants import MAX_TIMESTAMP_MS, MIN_TIMESTAMP_MS
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceContainerCRUD, ResourceCRUD
 from cognite_toolkit._cdf_tk.exceptions import (
@@ -41,11 +41,11 @@ from .data_organization import DataSetsCRUD
 
 
 @final
-class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries]):
+class TimeSeriesCRUD(ResourceContainerCRUD[ExternalId, TimeSeriesRequest, TimeSeriesResponse]):
     item_name = "datapoints"
     folder_name = "timeseries"
-    resource_cls = TimeSeries
-    resource_write_cls = TimeSeriesWrite
+    resource_cls = TimeSeriesResponse
+    resource_write_cls = TimeSeriesRequest
     yaml_cls = TimeSeriesYAML
     kind = "TimeSeries"
     dependencies = frozenset({DataSetsCRUD, GroupAllScopedCRUD, AssetCRUD})
@@ -57,7 +57,7 @@ class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries]):
 
     @classmethod
     def get_required_capability(
-        cls, items: Sequence[TimeSeriesWrite] | None, read_only: bool
+        cls, items: Sequence[TimeSeriesRequest] | None, read_only: bool
     ) -> Capability | list[Capability]:
         if not items and items is not None:
             return []
@@ -72,22 +72,22 @@ class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries]):
         return TimeSeriesAcl(actions, scope)
 
     @classmethod
-    def get_id(cls, item: TimeSeries | TimeSeriesWrite | dict) -> str:
+    def get_id(cls, item: TimeSeriesRequest | TimeSeriesResponse | dict) -> ExternalId:
         if isinstance(item, dict):
-            return item["externalId"]
+            return ExternalId(external_id=item["externalId"])
         if item.external_id is None:
             raise ToolkitRequiredValueError("TimeSeries must have external_id set.")
-        return item.external_id
+        return ExternalId(external_id=item.external_id)
 
     @classmethod
-    def get_internal_id(cls, item: TimeSeries | dict) -> int:
+    def get_internal_id(cls, item: TimeSeriesResponse | dict) -> int:
         if isinstance(item, dict):
             return item["id"]
         return item.id
 
     @classmethod
-    def dump_id(cls, id: str) -> dict[str, Any]:
-        return {"externalId": id}
+    def dump_id(cls, id: ExternalId) -> dict[str, Any]:
+        return id.dump()
 
     @classmethod
     def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
@@ -97,9 +97,9 @@ class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries]):
             for security_category in item["securityCategoryNames"]:
                 yield SecurityCategoryCRUD, security_category
         if "assetExternalId" in item:
-            yield AssetCRUD, item["assetExternalId"]
+            yield AssetCRUD, ExternalId(external_id=item["assetExternalId"])
 
-    def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> TimeSeriesWrite:
+    def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> TimeSeriesRequest:
         if ds_external_id := resource.pop("dataSetExternalId", None):
             resource["dataSetId"] = self.client.lookup.data_sets.id(ds_external_id, is_dry_run)
         if security_categories_names := resource.pop("securityCategoryNames", []):
@@ -108,10 +108,10 @@ class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries]):
             )
         if asset_external_id := resource.pop("assetExternalId", None):
             resource["assetId"] = self.client.lookup.assets.id(asset_external_id, is_dry_run)
-        return TimeSeriesWrite._load(resource)
+        return TimeSeriesRequest.model_validate(resource)
 
-    def dump_resource(self, resource: TimeSeries, local: dict[str, Any] | None = None) -> dict[str, Any]:
-        dumped = resource.as_write().dump()
+    def dump_resource(self, resource: TimeSeriesResponse, local: dict[str, Any] | None = None) -> dict[str, Any]:
+        dumped = resource.as_request_resource().dump()
         if data_set_id := dumped.pop("dataSetId", None):
             dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id)
         if security_categories := dumped.pop("securityCategories", []):
@@ -120,53 +120,48 @@ class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries]):
             dumped["assetExternalId"] = self.client.lookup.assets.external_id(asset_id)
         return dumped
 
-    def create(self, items: Sequence[TimeSeriesWrite]) -> TimeSeriesList:
-        return self.client.time_series.create(items)
+    def create(self, items: Sequence[TimeSeriesRequest]) -> list[TimeSeriesResponse]:
+        return self.client.tool.timeseries.create(items)
 
-    def retrieve(self, ids: SequenceNotStr[str | int]) -> TimeSeriesList:
-        internal_ids, external_ids = self._split_ids(ids)
-        return self.client.time_series.retrieve_multiple(
-            ids=internal_ids, external_ids=external_ids, ignore_unknown_ids=True
-        )
+    def retrieve(self, ids: SequenceNotStr[ExternalId]) -> list[TimeSeriesResponse]:
+        return self.client.tool.timeseries.retrieve(list(ids), ignore_unknown_ids=True)
 
-    def update(self, items: Sequence[TimeSeriesWrite]) -> TimeSeriesList:
-        return self.client.time_series.update(items, mode="replace")
+    def update(self, items: Sequence[TimeSeriesRequest]) -> list[TimeSeriesResponse]:
+        return self.client.tool.timeseries.update(items, mode="replace")
 
-    def delete(self, ids: SequenceNotStr[str | int]) -> int:
-        existing = self.retrieve(ids)
-        if existing:
-            self.client.time_series.delete(id=existing.as_ids(), ignore_unknown_ids=True)
-        return len(existing)
+    def delete(self, ids: SequenceNotStr[InternalOrExternalId]) -> int:
+        if not ids:
+            return 0
+        self.client.tool.timeseries.delete(list(ids), ignore_unknown_ids=True)
+        return len(ids)
 
     def _iterate(
         self,
         data_set_external_id: str | None = None,
         space: str | None = None,
         parent_ids: list[Hashable] | None = None,
-    ) -> Iterable[TimeSeries]:
-        return iter(
-            self.client.time_series(data_set_external_ids=[data_set_external_id] if data_set_external_id else None)
-        )
+    ) -> Iterable[TimeSeriesResponse]:
+        filter_ = ClassicFilter.from_asset_subtree_and_data_sets(data_set_id=data_set_external_id)
+        for timeseries in self.client.tool.timeseries.iterate(filter=filter_, limit=None):
+            yield from timeseries
 
-    def count(self, ids: str | dict[str, Any] | SequenceNotStr[str | dict[str, Any]] | None) -> int:
+    def count(self, ids: SequenceNotStr[ExternalId]) -> int:
         datapoints = self.client.time_series.data.retrieve(
-            external_id=ids,  # type: ignore[arg-type]
+            external_id=[id.external_id for id in ids],
             start=MIN_TIMESTAMP_MS,
             end=MAX_TIMESTAMP_MS + 1,
             aggregates="count",
             granularity="1000d",
             ignore_unknown_ids=True,
         )
-        return sum(sum(data.count or []) for data in datapoints)  # type: ignore[union-attr, misc, arg-type]
+        return sum(sum(data.count or []) for data in datapoints)
 
-    def drop_data(self, ids: SequenceNotStr[str] | None) -> int:
+    def drop_data(self, ids: SequenceNotStr[ExternalId]) -> int:
         count = self.count(ids)
-        existing = self.client.time_series.retrieve_multiple(
-            external_ids=cast(SequenceNotStr[str], ids), ignore_unknown_ids=True
-        ).as_external_ids()
-        for external_id in existing:
+        existing = self.client.tool.timeseries.retrieve(list(ids), ignore_unknown_ids=True)
+        for ts in existing:
             self.client.time_series.data.delete_range(
-                external_id=external_id, start=MIN_TIMESTAMP_MS, end=MAX_TIMESTAMP_MS + 1
+                external_id=ts.external_id, start=MIN_TIMESTAMP_MS, end=MAX_TIMESTAMP_MS + 1
             )
         return count
 
@@ -219,7 +214,7 @@ class DatapointSubscriptionCRUD(
         if "dataSetExternalId" in item:
             yield DataSetsCRUD, item["dataSetExternalId"]
         for timeseries_id in item.get("timeSeriesIds", []):
-            yield TimeSeriesCRUD, timeseries_id
+            yield TimeSeriesCRUD, ExternalId(external_id=timeseries_id)
 
     @classmethod
     def get_required_capability(

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py

@@ -65,7 +65,7 @@ from rich import print
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawDatabase, RawTable
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.raw import RawDatabase, RawTable
 from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
 from cognite_toolkit._cdf_tk.exceptions import (

cognite_toolkit/_cdf_tk/data_classes/__init__.py

@@ -29,6 +29,7 @@ from ._deploy_results import (
     ResourceDeployResult,
     UploadDeployResult,
 )
+from ._issues import Issue, IssueList
 from ._module_directories import ModuleDirectories, ModuleLocation
 from ._module_resources import ModuleResources
 from ._packages import Package, Packages
@@ -56,6 +57,8 @@ __all__ = [
     "DeployResults",
     "Environment",
     "InitConfigYAML",
+    "Issue",
+    "IssueList",
     "ModuleDirectories",
     "ModuleLocation",
     "ModuleResources",

cognite_toolkit/_cdf_tk/data_classes/_issues.py

@@ -0,0 +1,36 @@
+import sys
+from collections import UserList
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from pydantic import BaseModel, Field
+
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
+
+MODULE_ISSUE_CODE = "MOD"
+
+
+class Issue(BaseModel):
+    """Base class for all issues"""
+
+    code: str
+    message: str | None = Field(default=None)
+
+
+# temporary adapter to manage existing warnings
+class IssueList(UserList[Issue]):
+    """List of build issues."""
+
+    @classmethod
+    def from_warning_list(cls, warning_list: WarningList[ToolkitWarning]) -> Self:
+        """Create a IssueList from a WarningList."""
+        return cls([Issue(code="WARN", message=warning.get_message()) for warning in warning_list])
+
+
+class ModuleLoadingIssue(Issue):
+    """Issue with the loading of a module folder."""
+
+    code: str = "MOD_001"

cognite_toolkit/_cdf_tk/data_classes/_module_directories.py

@@ -8,7 +8,8 @@ from pathlib import Path
 from typing import Any, SupportsIndex, overload
 
 from cognite_toolkit._cdf_tk.constants import INDEX_PATTERN
-from cognite_toolkit._cdf_tk.utils import calculate_directory_hash, iterate_modules, resource_folder_from_path
+from cognite_toolkit._cdf_tk.utils.hashing import calculate_directory_hash
+from cognite_toolkit._cdf_tk.utils.modules import iterate_modules, resource_folder_from_path
 
 from ._module_toml import ModuleToml
 

cognite_toolkit/_cdf_tk/data_classes/_tracking_info.py

@@ -32,6 +32,10 @@ class CommandTrackingInfo(BaseModel):
     downloaded_library_ids: set[str] = Field(default_factory=set, alias="downloadedLibraryIds")
     downloaded_package_ids: set[str] = Field(default_factory=set, alias="downloadedPackageIds")
     downloaded_module_ids: set[str] = Field(default_factory=set, alias="downloadedModuleIds")
+    function_validation_count: int = Field(default=0, alias="functionValidationCount")
+    function_validation_failures: int = Field(default=0, alias="functionValidationFailures")
+    function_validation_credential_errors: int = Field(default=0, alias="functionValidationCredentialErrors")
+    function_validation_time_ms: int = Field(default=0, alias="functionValidationTimeMs")
 
     def to_dict(self) -> dict[str, Any]:
         """Convert the tracking info to a dictionary for Mixpanel.

cognite_toolkit/_cdf_tk/feature_flags.py

@@ -61,6 +61,14 @@ class Flags(Enum):
         visible=True,
         description="Enables extended upload to support uploading individual files",
     )
+    SIMULATORS = FlagMetadata(
+        visible=True,
+        description="Enables the support for simulator model resources",
+    )
+    FUNCTION_REQUIREMENTS_VALIDATION = FlagMetadata(
+        visible=True,
+        description="Enables validation of function requirements.txt during build using pip dry-run",
+    )
 
     def is_enabled(self) -> bool:
         return FeatureFlag.is_enabled(self)

cognite_toolkit/_cdf_tk/resource_classes/__init__.py

@@ -40,6 +40,7 @@ from .robotics import RobotCapabilityYAML, RobotDataPostProcessingYAML, RobotFra
 from .search_config import SearchConfigYAML
 from .securitycategories import SecurityCategoriesYAML
 from .sequence import SequenceRowYAML, SequenceYAML
+from .simulator_model import SimulatorModelYAML
 from .space import SpaceYAML
 from .streamlit_ import StreamlitYAML
 from .streams import StreamYAML
@@ -94,6 +95,7 @@ __all__ = [
     "SecurityCategoriesYAML",
     "SequenceRowYAML",
     "SequenceYAML",
+    "SimulatorModelYAML",
     "SpaceYAML",
     "StreamYAML",
     "StreamlitYAML",

cognite_toolkit/_cdf_tk/resource_classes/capabilities.py

@@ -510,6 +510,12 @@ class LegacyGenericsAcl(Capability):
     scope: AllScope
 
 
+class SimulatorsAcl(Capability):
+    _capability_name = "simulatorsAcl"
+    actions: list[Literal["READ", "WRITE", "DELETE", "RUN", "MANAGE"]]
+    scope: AllScope | DataSetScope
+
+
 class StreamsAcl(Capability):
     _capability_name = "streamsAcl"
     actions: list[Literal["READ", "CREATE", "DELETE"]]

cognite_toolkit/_cdf_tk/resource_classes/search_config.py

@@ -2,7 +2,7 @@ from typing import ClassVar
 
 from pydantic import Field
 
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.search_config import SearchConfigWrite
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.search_config import SearchConfigWrite
 
 from .base import BaseModelResource, ToolkitResource
 

cognite_toolkit/_cdf_tk/resource_classes/simulator_model.py

@@ -0,0 +1,17 @@
+from pydantic import Field
+
+from .base import ToolkitResource
+
+
+class SimulatorModelYAML(ToolkitResource):
+    """Simulator model YAML resource class.
+
+    Based on: https://api-docs.cognite.com/20230101/tag/Simulator-Models/operation/create_simulator_model_simulators_models_post
+    """
+
+    external_id: str = Field(description="External ID of the simulator model.", min_length=1, max_length=255)
+    simulator_external_id: str = Field(description="External id of the simulator.", min_length=1, max_length=50)
+    name: str = Field(description="The name of the simulator model.", min_length=1, max_length=50)
+    description: str | None = Field(None, description="Description of the simulator model.", max_length=500)
+    data_set_external_id: str = Field(description="The external ID of the dataset this simulator model belongs to.")
+    type: str = Field(description="The type of the simulator model.", min_length=1, max_length=50)

cognite_toolkit/_cdf_tk/resource_classes/workflow_version.py

@@ -93,10 +93,19 @@ class DynamicTaskParameters(BaseModelResource):
     dynamic: DynamicRef = Field(description="Reference to another task to use as the definition for this task.")
 
 
-class SubworkflowRef(BaseModelResource):
-    tasks: "WorkflowVersionId | list[Task]" = Field(
-        description="Reference to the subworkflow to execute. This can be either a reference to an existing workflow version or an inline definition of tasks."
-    )
+class SubworkflowInlineTasks(BaseModelResource):
+    """Inline definition of tasks for a subworkflow."""
+
+    tasks: "list[Task]" = Field(description="Inline definition of tasks for the subworkflow.")
+
+
+# SubworkflowRef can be either a reference to an existing workflow version OR an inline definition of tasks
+SubworkflowRef = Annotated[
+    WorkflowVersionId | SubworkflowInlineTasks,
+    Field(
+        description="Reference to the subworkflow to execute. This can be either a reference to an existing workflow version (with workflowExternalId and version) or an inline definition of tasks."
+    ),
+]
 
 
 class SubworkflowTaskParameters(BaseModelResource):

cognite_toolkit/_cdf_tk/storageio/_applications.py

@@ -1,16 +1,25 @@
 from collections.abc import Iterable, Sequence
+from itertools import zip_longest
 from typing import Any
 
+from pydantic import JsonValue
+
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.canvas import (
+from cognite_toolkit._cdf_tk.client.http_client import (
+    HTTPClient,
+    HTTPMessage,
+    HTTPResult2,
+    RequestMessage2,
+    SuccessResponse2,
+)
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.canvas import (
     IndustrialCanvas,
     IndustrialCanvasApply,
 )
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.charts import Chart, ChartList, ChartWrite
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.charts import Chart, ChartList, ChartWrite
 from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
 from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
 from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
-from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, HTTPMessage, SimpleBodyRequest
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
 from ._base import Page, UploadableStorageIO, UploadItem
@@ -185,23 +194,52 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
         results: list[HTTPMessage] = []
         for item in data_chunk:
             instances = item.item.as_instances()
-            items: list[dict[str, JsonVal]] = []
+            upsert_items: list[dict[str, JsonValue]] = []
             for instance in instances:
                 dumped = instance.dump()
                 if self.exclude_existing_version:
                     dumped.pop("existingVersion", None)
-                items.append(dumped)
-
-            responses = http_client.request_with_retries(
-                message=SimpleBodyRequest(
-                    endpoint_url=config.create_api_url("/models/instances"),
-                    method="POST",
-                    # MyPy does not understand that .dump is valid json
-                    body_content={"items": items},  # type: ignore[dict-item]
-                )
-            )
-            results.extend(responses.as_item_responses(item.source_id))
+                upsert_items.append(dumped)
+
+            canvas = item.item.canvas
+            existing = self.client.canvas.industrial.retrieve(canvas.external_id)
+            if existing is not None:
+                existing_instance_ids = existing.as_write().as_instance_ids(include_solution_tags=False)
+                delete_set = set(existing_instance_ids) - set(item.item.as_instance_ids())
+                to_delete: list[Any] = [
+                    {
+                        "space": instance_id.space,
+                        "externalId": instance_id.external_id,
+                        "instanceType": instance_id.instance_type,
+                    }
+                    for instance_id in delete_set
+                ]
+            else:
+                to_delete = []
 
+            last_response: HTTPResult2 | None = None
+            for upsert_chunk, delete_chunk in zip_longest(
+                chunker_sequence(upsert_items, 1000), chunker_sequence(to_delete, 1000), fillvalue=None
+            ):
+                body_content: dict[str, JsonValue] = {}
+                if upsert_chunk:
+                    # MyPy fails do understand that list[dict[str, JsonValue]] is a subtype of JsonValue
+                    body_content["items"] = upsert_chunk  # type: ignore[assignment]
+                if delete_chunk:
+                    body_content["delete"] = delete_chunk
+
+                response = http_client.request_single_retries(
+                    message=RequestMessage2(
+                        endpoint_url=config.create_api_url("/models/instances"),
+                        method="POST",
+                        body_content=body_content,
+                    )
+                )
+                if not isinstance(response, SuccessResponse2):
+                    results.append(response.as_item_response(item.source_id))
+                last_response = response
+            if last_response is not None:
+                results.append(last_response.as_item_response(item.source_id))
         return results
 
     def data_to_json_chunk(