cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/__init__.py

@@ -0,0 +1,98 @@
+from .agent import AgentCRUD
+from .auth import GroupAllScopedCRUD, GroupCRUD, SecurityCategoryCRUD
+from .classic import AssetCRUD, EventCRUD, SequenceCRUD, SequenceRowCRUD
+from .configuration import SearchConfigCRUD
+from .data_organization import DataSetsCRUD, LabelCRUD
+from .datamodel import (
+    ContainerCRUD,
+    DataModelCRUD,
+    EdgeCRUD,
+    GraphQLCRUD,
+    NodeCRUD,
+    SpaceCRUD,
+    ViewCRUD,
+)
+from .extraction_pipeline import ExtractionPipelineConfigCRUD, ExtractionPipelineCRUD
+from .fieldops import InFieldCDMLocationConfigCRUD, InFieldLocationConfigCRUD, InfieldV1CRUD
+from .file import CogniteFileCRUD, FileMetadataCRUD
+from .function import FunctionCRUD, FunctionScheduleCRUD
+from .group_scoped import GroupResourceScopedCRUD
+from .hosted_extractors import (
+    HostedExtractorDestinationCRUD,
+    HostedExtractorJobCRUD,
+    HostedExtractorMappingCRUD,
+    HostedExtractorSourceCRUD,
+)
+from .industrial_tool import StreamlitCRUD
+from .location import LocationFilterCRUD
+from .migration import ResourceViewMappingCRUD
+from .raw import RawDatabaseCRUD, RawTableCRUD
+from .relationship import RelationshipCRUD
+from .robotics import (
+    RobotCapabilityCRUD,
+    RoboticFrameCRUD,
+    RoboticLocationCRUD,
+    RoboticMapCRUD,
+    RoboticsDataPostProcessingCRUD,
+)
+from .streams import StreamCRUD
+from .three_d_model import ThreeDModelCRUD
+from .timeseries import DatapointSubscriptionCRUD, TimeSeriesCRUD
+from .transformation import TransformationCRUD, TransformationNotificationCRUD, TransformationScheduleCRUD
+from .workflow import WorkflowCRUD, WorkflowTriggerCRUD, WorkflowVersionCRUD
+
+__all__ = [
+    "AgentCRUD",
+    "AssetCRUD",
+    "CogniteFileCRUD",
+    "ContainerCRUD",
+    "DataModelCRUD",
+    "DataSetsCRUD",
+    "DatapointSubscriptionCRUD",
+    "EdgeCRUD",
+    "EventCRUD",
+    "ExtractionPipelineCRUD",
+    "ExtractionPipelineConfigCRUD",
+    "FileMetadataCRUD",
+    "FunctionCRUD",
+    "FunctionScheduleCRUD",
+    "GraphQLCRUD",
+    "GroupAllScopedCRUD",
+    "GroupCRUD",
+    "GroupResourceScopedCRUD",
+    "HostedExtractorDestinationCRUD",
+    "HostedExtractorJobCRUD",
+    "HostedExtractorMappingCRUD",
+    "HostedExtractorSourceCRUD",
+    "InFieldCDMLocationConfigCRUD",
+    "InFieldLocationConfigCRUD",
+    "InfieldV1CRUD",
+    "LabelCRUD",
+    "LocationFilterCRUD",
+    "NodeCRUD",
+    "RawDatabaseCRUD",
+    "RawTableCRUD",
+    "RelationshipCRUD",
+    "ResourceViewMappingCRUD",
+    "RobotCapabilityCRUD",
+    "RoboticFrameCRUD",
+    "RoboticLocationCRUD",
+    "RoboticMapCRUD",
+    "RoboticsDataPostProcessingCRUD",
+    "SearchConfigCRUD",
+    "SecurityCategoryCRUD",
+    "SequenceCRUD",
+    "SequenceRowCRUD",
+    "SpaceCRUD",
+    "StreamCRUD",
+    "StreamlitCRUD",
+    "ThreeDModelCRUD",
+    "TimeSeriesCRUD",
+    "TransformationCRUD",
+    "TransformationNotificationCRUD",
+    "TransformationScheduleCRUD",
+    "ViewCRUD",
+    "WorkflowCRUD",
+    "WorkflowTriggerCRUD",
+    "WorkflowVersionCRUD",
+]

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/agent.py

@@ -0,0 +1,115 @@
+from collections.abc import Hashable, Iterable, Sequence
+from typing import Any
+
+from cognite.client.data_classes.agents import Agent, AgentList, AgentUpsert
+from cognite.client.data_classes.capabilities import AgentsAcl, Capability
+from cognite.client.exceptions import CogniteAPIError
+from cognite.client.utils.useful_types import SequenceNotStr
+
+from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
+from cognite_toolkit._cdf_tk.resource_classes import AgentYAML
+from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_hashable, diff_list_identifiable
+
+
+class AgentCRUD(ResourceCRUD[str, AgentUpsert, Agent]):
+    folder_name = "agents"
+    resource_cls = Agent
+    resource_write_cls = AgentUpsert
+    kind = "Agent"
+    yaml_cls = AgentYAML
+    _doc_base_url = ""
+    _doc_url = "https://api-docs.cognite.com/20230101-beta/tag/Agents/operation/main_ai_agents_post/"
+
+    @classmethod
+    def get_id(cls, item: AgentUpsert | Agent | dict) -> str:
+        if isinstance(item, dict):
+            return item["externalId"]
+        return item.external_id
+
+    @classmethod
+    def dump_id(cls, id: str) -> dict[str, Any]:
+        return {"externalId": id}
+
+    @classmethod
+    def get_required_capability(
+        cls, items: Sequence[AgentUpsert] | None, read_only: bool
+    ) -> Capability | list[Capability]:
+        if not items and items is not None:
+            return []
+
+        actions = [AgentsAcl.Action.READ] if read_only else [AgentsAcl.Action.READ, AgentsAcl.Action.WRITE]
+
+        return AgentsAcl(actions, AgentsAcl.Scope.All())
+
+    def create(self, items: Sequence[AgentUpsert]) -> AgentList:
+        return self.client.agents.upsert(items)
+
+    def retrieve(self, ids: SequenceNotStr[str]) -> AgentList:
+        return self.client.agents.retrieve(ids, ignore_unknown_ids=True)
+
+    def update(self, items: Sequence[AgentUpsert]) -> AgentList:
+        return self.client.agents.upsert(items)
+
+    def delete(self, ids: SequenceNotStr[str]) -> int:
+        try:
+            self.client.agents.delete(ids)
+        except CogniteAPIError:
+            deleted = 0
+            for id in ids:
+                try:
+                    self.client.agents.delete(id)
+                    deleted += 1
+                except CogniteAPIError:
+                    # accepted because the resource may not exist
+                    pass
+            return deleted
+        return len(ids)
+
+    def _iterate(
+        self,
+        data_set_external_id: str | None = None,
+        space: str | None = None,
+        parent_ids: list[Hashable] | None = None,
+    ) -> Iterable[Agent]:
+        return self.client.agents.list()
+
+    def dump_resource(self, resource: Agent, local: dict[str, Any] | None = None) -> dict[str, Any]:
+        dumped = resource.as_write().dump()
+        # The atlas endpoints are not yet full implemented. There are properties being added and removed that are
+        # not part of the official API. For example, as of 31.July 2025, `labels` is not part of the API, however,
+        # this is necessary to ensure that the agents are shown as published in the UI, so we cannot ignore it.
+        # The below logic ensures that we keep the unknown properties returned by the API, such that when we run
+        # `cdf dump agents` we will not lose any properties that are not part of the official API.
+        if (unknown_props := getattr(resource, "_unknown_properties", None)) and isinstance(unknown_props, dict):
+            dumped.update(unknown_props)
+        if local is None:
+            return dumped
+        if resource.instructions == "" and "instructions" not in local:
+            # Instructions are optional, if not set the server set them to an empty string.
+            # We remove them from the dumped resource to ensure it will be equal to the local resource.
+            dumped.pop("instructions", None)
+        for key in ["labels", "exampleQuestions"]:
+            if key not in local and not dumped.get(key):
+                # If the local resource does not have the key and the server set Agent has it set to an empty list,
+                # we remove it from the dumped resource to ensure it will be equal to the local resource.
+                dumped.pop(key, None)
+        return dumped
+
+    def diff_list(
+        self, local: list[Any], cdf: list[Any], json_path: tuple[str | int, ...]
+    ) -> tuple[dict[int, int], list[int]]:
+        """
+        Compare two lists and return a mapping of local indices to CDF indices and a list of CDF indices that are not
+        present in the local list.
+        """
+        if json_path == ("tools",):
+            return diff_list_identifiable(
+                local, cdf, get_identifier=lambda t: (t.get("name", ""), t.get("description", ""))
+            )
+        elif json_path == ("labels",):
+            return diff_list_hashable(local, cdf)
+        elif json_path == ("exampleQuestions",):
+            return diff_list_identifiable(
+                local, cdf, get_identifier=lambda q: q.get("question", "") if isinstance(q, dict) else str(q)
+            )
+        return super().diff_list(local, cdf, json_path)

cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/auth_loaders.py → cruds/_resource_cruds/auth.py}

@@ -11,13 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from __future__ import annotations
+
 
 import difflib
 from collections import defaultdict
 from collections.abc import Callable, Hashable, Iterable, Sequence
 from dataclasses import dataclass
-from functools import lru_cache
 from pathlib import Path
 from typing import Any, Literal, cast, final
 
@@ -31,11 +30,9 @@ from cognite.client.data_classes.iam import (
     Group,
     GroupList,
     GroupWrite,
-    GroupWriteList,
     SecurityCategory,
     SecurityCategoryList,
     SecurityCategoryWrite,
-    SecurityCategoryWriteList,
 )
 from cognite.client.exceptions import CogniteAPIError, CogniteNotFoundError
 from cognite.client.utils.useful_types import SequenceNotStr
@@ -43,12 +40,11 @@ from rich import print
 from rich.console import Console
 from rich.markup import escape
 
-from cognite_toolkit._cdf_tk._parameters import ANY_INT, ANY_STR, ANYTHING, ParameterSpec, ParameterSpecSet
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.raw import RawDatabase, RawTable
+from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
 from cognite_toolkit._cdf_tk.exceptions import ToolkitWrongResourceError
-from cognite_toolkit._cdf_tk.loaders._base_loaders import ResourceLoader
-from cognite_toolkit._cdf_tk.resource_classes import GroupYAML
+from cognite_toolkit._cdf_tk.resource_classes import GroupYAML, SecurityCategoriesYAML
 from cognite_toolkit._cdf_tk.tk_warnings import (
     HighSeverityWarning,
     MediumSeverityWarning,
@@ -62,19 +58,16 @@ class _ReplaceMethod:
     """This is a small helper class used in the
     lookup and replace in the ACL scoped ids"""
 
-    lookup_method: Callable[[str, bool], int]
+    lookup_method: Callable[[str, bool], int | None]
     reverse_lookup_method: Callable[[int], str | None]
     id_name: str
 
 
-class GroupLoader(ResourceLoader[str, GroupWrite, Group, GroupWriteList, GroupList]):
+class GroupCRUD(ResourceCRUD[str, GroupWrite, Group]):
     folder_name = "auth"
-    filename_pattern = r"^(?!.*SecurityCategory$).*"
     kind = "Group"
     resource_cls = Group
     resource_write_cls = GroupWrite
-    list_cls = GroupList
-    list_write_cls = GroupWriteList
     yaml_cls = GroupYAML
     resource_scopes = frozenset(
         {
@@ -145,14 +138,14 @@ class GroupLoader(ResourceLoader[str, GroupWrite, Group, GroupWriteList, GroupLi
         return {"name": id}
 
     @classmethod
-    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader], Hashable]]:
-        from .classic_loaders import AssetLoader
-        from .data_organization_loaders import DataSetsLoader
-        from .datamodel_loaders import SpaceLoader
-        from .extraction_pipeline_loaders import ExtractionPipelineLoader
-        from .location_loaders import LocationFilterLoader
-        from .raw_loaders import RawDatabaseLoader, RawTableLoader
-        from .timeseries_loaders import TimeSeriesLoader
+    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
+        from .classic import AssetCRUD
+        from .data_organization import DataSetsCRUD
+        from .datamodel import SpaceCRUD
+        from .extraction_pipeline import ExtractionPipelineCRUD
+        from .location import LocationFilterCRUD
+        from .raw import RawDatabaseCRUD, RawTableCRUD
+        from .timeseries import TimeSeriesCRUD
 
         for capability in item.get("capabilities", []):
             for acl, content in capability.items():
@@ -160,41 +153,41 @@ class GroupLoader(ResourceLoader[str, GroupWrite, Group, GroupWriteList, GroupLi
                     if space_ids := scope.get(cap.SpaceIDScope._scope_name, []):
                         if isinstance(space_ids, dict) and "spaceIds" in space_ids:
                             for space_id in space_ids["spaceIds"]:
-                                yield SpaceLoader, space_id
+                                yield SpaceCRUD, space_id
                     if data_set_ids := scope.get(cap.DataSetScope._scope_name, []):
                         if isinstance(data_set_ids, dict) and "ids" in data_set_ids:
                             for data_set_id in data_set_ids["ids"]:
-                                yield DataSetsLoader, data_set_id
+                                yield DataSetsCRUD, data_set_id
                     if table_ids := scope.get(cap.TableScope._scope_name, []):
                         for db_name, tables in table_ids.get("dbsToTables", {}).items():
-                            yield RawDatabaseLoader, RawDatabase(db_name)
+                            yield RawDatabaseCRUD, RawDatabase(db_name)
                             if isinstance(tables, list):
-                                yield from ((RawTableLoader, RawTable(db_name, table)) for table in tables)
+                                yield from ((RawTableCRUD, RawTable(db_name, table)) for table in tables)
                             elif isinstance(tables, dict) and "tables" in tables:
                                 for table in tables["tables"]:
-                                    yield RawTableLoader, RawTable(db_name, table)
+                                    yield RawTableCRUD, RawTable(db_name, table)
                     if extraction_pipeline_ids := scope.get(cap.ExtractionPipelineScope._scope_name, []):
                         if isinstance(extraction_pipeline_ids, dict) and "ids" in extraction_pipeline_ids:
                             for extraction_pipeline_id in extraction_pipeline_ids["ids"]:
-                                yield ExtractionPipelineLoader, extraction_pipeline_id
+                                yield ExtractionPipelineCRUD, extraction_pipeline_id
                     if asset_root_ids := scope.get(cap.AssetRootIDScope._scope_name, []):
                         if isinstance(asset_root_ids, dict) and "rootIds" in asset_root_ids:
                             for asset_root_id in asset_root_ids["rootIds"]:
-                                yield AssetLoader, asset_root_id
+                                yield AssetCRUD, asset_root_id
                     if (ids := scope.get(cap.IDScope._scope_name, [])) or (
                         ids := scope.get(cap.IDScopeLowerCase._scope_name, [])
                     ):
-                        loader: type[ResourceLoader] | None = None
+                        loader: type[ResourceCRUD] | None = None
                         if acl == cap.DataSetsAcl._capability_name:
-                            loader = DataSetsLoader
+                            loader = DataSetsCRUD
                         elif acl == cap.ExtractionPipelinesAcl._capability_name:
-                            loader = ExtractionPipelineLoader
+                            loader = ExtractionPipelineCRUD
                         elif acl == cap.TimeSeriesAcl._capability_name:
-                            loader = TimeSeriesLoader
+                            loader = TimeSeriesCRUD
                         elif acl == cap.SecurityCategoriesAcl._capability_name:
-                            loader = SecurityCategoryLoader
+                            loader = SecurityCategoryCRUD
                         elif acl == cap.LocationFiltersAcl._capability_name:
-                            loader = LocationFilterLoader
+                            loader = LocationFilterCRUD
                         if loader is not None and isinstance(ids, dict) and "ids" in ids:
                             for id_ in ids["ids"]:
                                 yield loader, id_
@@ -208,7 +201,7 @@ class GroupLoader(ResourceLoader[str, GroupWrite, Group, GroupWriteList, GroupLi
                 if len(scope) != 1:
                     # This will raise an error when the group is loaded.
                     continue
-                scope_name, scope_content = next(iter(scope.items()))
+                scope_name, _scope_content = next(iter(scope.items()))
 
                 if (acl, scope_name) in replace_method_by_acl:
                     replace_method = replace_method_by_acl[(acl, scope_name)]
@@ -457,32 +450,6 @@ class GroupLoader(ResourceLoader[str, GroupWrite, Group, GroupWriteList, GroupLi
     ) -> Iterable[Group]:
         return self.client.iam.groups.list(all=True)
 
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        spec = super().get_write_cls_parameter_spec()
-        # The Capability class in the SDK class Group implementation is deviating from the API.
-        # So we need to modify the spec to match the API.
-        for item in spec:
-            if item.path[0] == "capabilities" and len(item.path) > 2:
-                # Add extra ANY_STR layer
-                # The spec class is immutable, so we use this trick to modify it.
-                object.__setattr__(item, "path", item.path[:2] + (ANY_STR,) + item.path[2:])
-        spec.add(
-            ParameterSpec(
-                ("capabilities", ANY_INT, ANY_STR), frozenset({"dict"}), is_required=False, _is_nullable=False
-            )
-        )
-        spec.add(
-            ParameterSpec(
-                ("capabilities", ANY_INT, ANY_STR, "scope", ANYTHING),
-                frozenset({"dict"}),
-                is_required=True,
-                _is_nullable=False,
-            )
-        )
-        return spec
-
     def diff_list(
         self, local: list[Any], cdf: list[Any], json_path: tuple[str | int, ...]
     ) -> tuple[dict[int, int], list[int]]:
@@ -497,7 +464,7 @@ class GroupLoader(ResourceLoader[str, GroupWrite, Group, GroupWriteList, GroupLi
 
 
 @final
-class GroupAllScopedLoader(GroupLoader):
+class GroupAllScopedCRUD(GroupCRUD):
     def __init__(self, client: ToolkitClient, build_dir: Path | None, console: Console | None):
         super().__init__(client, build_dir, console, "all_scoped_only")
 
@@ -507,17 +474,13 @@ class GroupAllScopedLoader(GroupLoader):
 
 
 @final
-class SecurityCategoryLoader(
-    ResourceLoader[str, SecurityCategoryWrite, SecurityCategory, SecurityCategoryWriteList, SecurityCategoryList]
-):
-    filename_pattern = r"^.*SecurityCategory$"  # Matches all yaml files who's stem ends with *SecurityCategory.
+class SecurityCategoryCRUD(ResourceCRUD[str, SecurityCategoryWrite, SecurityCategory]):
     resource_cls = SecurityCategory
     resource_write_cls = SecurityCategoryWrite
-    list_cls = SecurityCategoryList
-    list_write_cls = SecurityCategoryWriteList
     kind = "SecurityCategory"
+    yaml_cls = SecurityCategoriesYAML
     folder_name = "auth"
-    dependencies = frozenset({GroupAllScopedLoader})
+    dependencies = frozenset({GroupAllScopedCRUD})
     _doc_url = "Security-categories/operation/createSecurityCategories"
 
     @property
@@ -561,7 +524,7 @@ class SecurityCategoryLoader(
             SecurityCategoriesAcl.Scope.All(),
         )
 
-    def create(self, items: SecurityCategoryWriteList) -> SecurityCategoryList:
+    def create(self, items: Sequence[SecurityCategoryWrite]) -> SecurityCategoryList:
         return self.client.iam.security_categories.create(items)
 
     def retrieve(self, ids: SequenceNotStr[str]) -> SecurityCategoryList:
@@ -569,7 +532,7 @@ class SecurityCategoryLoader(
         categories = self.client.iam.security_categories.list(limit=-1)
         return SecurityCategoryList([c for c in categories if c.name in names])
 
-    def update(self, items: SecurityCategoryWriteList) -> SecurityCategoryList:
+    def update(self, items: Sequence[SecurityCategoryWrite]) -> SecurityCategoryList:
         items_by_name = {item.name: item for item in items}
         retrieved = self.retrieve(list(items_by_name.keys()))
         retrieved_by_name = {item.name: item for item in retrieved}