cognite-toolkit 0.6.39__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_cdf_tk/cruds/__init__.py

@@ -41,6 +41,8 @@ from ._resource_cruds import (
     HostedExtractorJobCRUD,
     HostedExtractorMappingCRUD,
     HostedExtractorSourceCRUD,
+    InFieldCDMLocationConfigCRUD,
+    InFieldLocationConfigCRUD,
     InfieldV1CRUD,
     LabelCRUD,
     LocationFilterCRUD,
@@ -59,6 +61,7 @@ from ._resource_cruds import (
     SequenceCRUD,
     SequenceRowCRUD,
     SpaceCRUD,
+    StreamCRUD,
     StreamlitCRUD,
     ThreeDModelCRUD,
     TimeSeriesCRUD,
@@ -75,14 +78,14 @@ from ._worker import ResourceWorker
 _EXCLUDED_CRUDS: set[type[ResourceCRUD]] = set()
 if not FeatureFlag.is_enabled(Flags.GRAPHQL):
     _EXCLUDED_CRUDS.add(GraphQLCRUD)
-if not FeatureFlag.is_enabled(Flags.AGENTS):
-    _EXCLUDED_CRUDS.add(AgentCRUD)
 if not FeatureFlag.is_enabled(Flags.INFIELD):
     _EXCLUDED_CRUDS.add(InfieldV1CRUD)
+    _EXCLUDED_CRUDS.add(InFieldLocationConfigCRUD)
+    _EXCLUDED_CRUDS.add(InFieldCDMLocationConfigCRUD)
 if not FeatureFlag.is_enabled(Flags.MIGRATE):
     _EXCLUDED_CRUDS.add(ResourceViewMappingCRUD)
-if not FeatureFlag.is_enabled(Flags.SEARCH_CONFIG):
-    _EXCLUDED_CRUDS.add(SearchConfigCRUD)
+if not FeatureFlag.is_enabled(Flags.STREAMS):
+    _EXCLUDED_CRUDS.add(StreamCRUD)
 
 
 CRUDS_BY_FOLDER_NAME: dict[str, list[type[Loader]]] = {}
@@ -101,6 +104,14 @@ for _loader in itertools.chain(
     CRUDS_BY_FOLDER_NAME[_loader.folder_name].append(_loader)  # type: ignore[arg-type, attr-defined]
 del _loader  # cleanup module namespace
 
+# For backwards compatibility
+CRUDS_BY_FOLDER_NAME["data_models"] = CRUDS_BY_FOLDER_NAME["data_modeling"]  # Todo: Remove in v1.0
+RESOURCE_CRUD_BY_FOLDER_NAME = {
+    folder_name: cruds
+    for folder_name, loaders in CRUDS_BY_FOLDER_NAME.items()
+    if (cruds := [crud for crud in loaders if issubclass(crud, ResourceCRUD)])
+}
+
 CRUD_LIST = list(itertools.chain.from_iterable(CRUDS_BY_FOLDER_NAME.values()))
 RESOURCE_CRUD_LIST = [loader for loader in CRUD_LIST if issubclass(loader, ResourceCRUD)]
 RESOURCE_CRUD_CONTAINER_LIST = [loader for loader in CRUD_LIST if issubclass(loader, ResourceContainerCRUD)]
@@ -118,7 +129,8 @@ ResourceTypes: TypeAlias = Literal[
     "auth",
     "cdf_applications",
     "classic",
-    "data_models",
+    "data_modeling",
+    "data_models",  # Todo: Remove in v1.0
     "data_sets",
     "hosted_extractors",
     "locations",
@@ -130,6 +142,7 @@ ResourceTypes: TypeAlias = Literal[
     "functions",
     "raw",
     "robotics",
+    "streams",
     "streamlit",
     "workflows",
 ]
@@ -146,9 +159,11 @@ __all__ = [
     "CRUDS_BY_FOLDER_NAME",
     "CRUD_LIST",
     "KINDS_BY_FOLDER_NAME",
+    "RESOURCE_CRUD_BY_FOLDER_NAME",
     "RESOURCE_CRUD_CONTAINER_LIST",
     "RESOURCE_CRUD_LIST",
     "RESOURCE_DATA_CRUD_LIST",
+    "AgentCRUD",
     "AssetCRUD",
     "CogniteFileCRUD",
     "ContainerCRUD",
@@ -172,6 +187,8 @@ __all__ = [
     "HostedExtractorJobCRUD",
     "HostedExtractorMappingCRUD",
     "HostedExtractorSourceCRUD",
+    "InFieldCDMLocationConfigCRUD",
+    "InFieldLocationConfigCRUD",
     "LabelCRUD",
     "LocationFilterCRUD",
     "NodeCRUD",

cognite_toolkit/_cdf_tk/cruds/_base_cruds.py

@@ -1,27 +1,20 @@
-import re
 import sys
 from abc import ABC, abstractmethod
-from collections.abc import Hashable, Iterable, Sequence, Set, Sized
-from functools import lru_cache
+from collections.abc import Hashable, Iterable, Sequence, Sized
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generic, TypeVar
 
-from cognite.client.data_classes._base import (
-    T_CogniteResourceList,
-    T_WritableCogniteResource,
-    T_WriteClass,
-)
 from cognite.client.data_classes.capabilities import Capability
 from cognite.client.utils.useful_types import SequenceNotStr
 from rich.console import Console
 
-from cognite_toolkit._cdf_tk._parameters import ParameterSpecSet, read_parameter_from_init_type_hints
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING, EXCL_FILES, USE_SENTRY
+from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING, YAML_SUFFIX
+from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.resource_classes import ToolkitResource
 from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning
-from cognite_toolkit._cdf_tk.utils import load_yaml_inject_variables, safe_read, to_directory_compatible
-from cognite_toolkit._cdf_tk.utils.useful_types import T_ID, T_WritableCogniteResourceList
+from cognite_toolkit._cdf_tk.utils import load_yaml_inject_variables, safe_read, sanitize_filename
+from cognite_toolkit._cdf_tk.utils.useful_types import T_ID
 
 if TYPE_CHECKING:
     from cognite_toolkit._cdf_tk.data_classes import BuildEnvironment
@@ -32,9 +25,6 @@ else:
     from typing_extensions import Self
 
 
-_COMPILED_PATTERN: dict[str, re.Pattern] = {}
-
-
 class Loader(ABC):
     """This is the base class for all loaders
 
@@ -45,20 +35,16 @@ class Loader(ABC):
     Class attributes:
         filetypes: The filetypes that are supported by this loader. This should be set in all subclasses.
         folder_name: The name of the folder in the build directory where the files are located. This should be set in all subclasses.
-        filename_pattern: A regex pattern that is used to filter the files that are supported by this loader. This is used
-            when two loaders have the same folder name to differentiate between them. If not set, all files are supported.
         dependencies: A set of loaders that must be loaded before this loader.
         exclude_filetypes: A set of filetypes that should be excluded from the supported filetypes.
     """
 
-    filetypes: frozenset[str]
     folder_name: str
     kind: str
-    filename_pattern: str = ""
     dependencies: "frozenset[type[ResourceCRUD]]" = frozenset()
-    exclude_filetypes: frozenset[str] = frozenset()
     _doc_base_url: str = "https://api-docs.cognite.com/20230101/tag/"
     _doc_url: str = ""
+    sub_folder_name: str | None = None
 
     def __init__(self, client: ToolkitClient, build_dir: Path | None, console: Console | None = None) -> None:
         self.client = client
@@ -86,13 +72,12 @@ class Loader(ABC):
     def doc_url(cls) -> str:
         return cls._doc_base_url + cls._doc_url
 
-    def find_files(self, dir_or_file: Path | None = None, include_formats: Set[str] | None = None) -> list[Path]:
+    def find_files(self, dir_or_file: Path | None = None) -> list[Path]:
         """Find all files that are supported by this loader in the given directory or file.
 
         Args:
             dir_or_file (Path): The directory or file to search in. If no path is given,
                 the build directory is used.
-            include_formats (set[str]): A set of file formats to include. If not set, all formats are included.
 
         Returns:
             list[Path]: A sorted list of all files that are supported by this loader.
@@ -106,12 +91,7 @@ class Loader(ABC):
                 raise ValueError("Invalid file type")
             return [dir_or_file]
         elif dir_or_file.is_dir():
-            file_paths = [
-                file
-                for file in dir_or_file.glob("**/*")
-                if self.is_supported_file(file) and (include_formats is None or file.suffix in include_formats)
-            ]
-            return sorted(file_paths)
+            return sorted([file for file in dir_or_file.rglob("*") if self.is_supported_file(file)])
         else:
             return []
 
@@ -120,41 +100,23 @@ class Loader(ABC):
         return any(cls.is_supported_file(file) for file in directory.glob("**/*"))
 
     @classmethod
-    def is_supported_file(cls, file: Path, force_pattern: bool = False) -> bool:
+    def is_supported_file(cls, file: Path) -> bool:
         """Check if hte file is supported by this loader.
 
         Args:
             file: The filepath to check.
-            force_pattern: If True, the filename pattern is used to determine if the file is supported. If False, the
-                file extension is used to determine if the file is supported (given that the
-                RequireKind flag is enabled).
 
         Returns:
             bool: True if the file is supported, False otherwise.
 
         """
-        if cls.filetypes and file.suffix[1:] not in cls.filetypes:
-            return False
-        if cls.exclude_filetypes and file.suffix[1:] in cls.exclude_filetypes:
-            return False
-        if force_pattern is False and not issubclass(cls, DataCRUD):
-            return file.stem.casefold().endswith(cls.kind.casefold())
-        else:
-            if cls.filename_pattern:
-                if cls.filename_pattern not in _COMPILED_PATTERN:
-                    _COMPILED_PATTERN[cls.filename_pattern] = re.compile(cls.filename_pattern, re.IGNORECASE)
-                return _COMPILED_PATTERN[cls.filename_pattern].match(file.stem) is not None
-        return True
+        return file.suffix in YAML_SUFFIX and file.stem.casefold().endswith(cls.kind.casefold())
 
 
 T_Loader = TypeVar("T_Loader", bound=Loader)
 
 
-class ResourceCRUD(
-    Loader,
-    ABC,
-    Generic[T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList],
-):
+class ResourceCRUD(Loader, ABC, Generic[T_ID, T_ResourceRequest, T_ResourceResponse]):
     """This is the base class for all resource CRUD.
 
     A resource loader consists of the following
@@ -165,26 +127,23 @@ class ResourceCRUD(
     All resources supported by the cognite_toolkit should implement a CRUD.
 
     Class attributes:
-        resource_write_cls: The write data class for the resource.
-        resource_cls: The read data class for the resource.
-        list_cls: The read list format for this resource.
-        list_write_cls: The write list format for this resource.
+        resource_write_cls: The API write data class for the resource.
+        resource_cls: The API read data class for the resource.
+        yaml_cls: The File format for this resource. This is used to validate the user input.
         support_drop: Whether the resource supports the drop flag.
-        filetypes: The filetypes that are supported by this crud. This should not be set in the subclass, it
-            should always be yaml and yml.
+        support_update: Whether the resource supports the update operation.
         dependencies: A set of other resource cruds that must be loaded before this crud.
+        parent_resource: A set of other resource cruds that are parent resources to this resource. This is used
+            to determine if the iterate method should return any resources when filtering by parent ids.
     """
 
     # Must be set in the subclass
-    resource_write_cls: type[T_WriteClass]
-    resource_cls: type[T_WritableCogniteResource]
-    list_cls: type[T_WritableCogniteResourceList]
-    list_write_cls: type[T_CogniteResourceList]
-    yaml_cls: type[ToolkitResource] | None = None
+    resource_write_cls: type[T_ResourceRequest]
+    resource_cls: type[T_ResourceResponse]
+    yaml_cls: type[ToolkitResource]
     # Optional to set in the subclass
     support_drop = True
     support_update = True
-    filetypes = frozenset({"yaml", "yml"})
     dependencies: "frozenset[type[ResourceCRUD]]" = frozenset()
     # For example, TransformationNotification and Schedule has Transformation as the parent resource
     # This is used in the iterate method to ensure that nothing is returned if
@@ -194,7 +153,7 @@ class ResourceCRUD(
     # The methods that must be implemented in the subclass
     @classmethod
     @abstractmethod
-    def get_id(cls, item: T_WriteClass | T_WritableCogniteResource | dict) -> T_ID:
+    def get_id(cls, item: T_ResourceRequest | T_ResourceResponse | dict) -> T_ID:
         raise NotImplementedError
 
     @classmethod
@@ -205,19 +164,19 @@ class ResourceCRUD(
     @classmethod
     @abstractmethod
     def get_required_capability(
-        cls, items: Sequence[T_WriteClass] | None, read_only: bool
+        cls, items: Sequence[T_ResourceRequest] | None, read_only: bool
     ) -> Capability | list[Capability]:
         raise NotImplementedError(f"get_required_capability must be implemented for {cls.__name__}.")
 
     @abstractmethod
-    def create(self, items: T_CogniteResourceList) -> Sized:
+    def create(self, items: Sequence[T_ResourceRequest]) -> Sized:
         raise NotImplementedError
 
     @abstractmethod
-    def retrieve(self, ids: SequenceNotStr[T_ID]) -> T_WritableCogniteResourceList:
+    def retrieve(self, ids: SequenceNotStr[T_ID]) -> Sequence[T_ResourceResponse]:
         raise NotImplementedError
 
-    def update(self, items: T_CogniteResourceList) -> Sized:
+    def update(self, items: Sequence[T_ResourceRequest]) -> Sized:
         raise NotImplementedError(f"Update is not supported for {type(self).__name__}.")
 
     @abstractmethod
@@ -229,7 +188,7 @@ class ResourceCRUD(
         data_set_external_id: str | None = None,
         space: str | None = None,
         parent_ids: list[Hashable] | None = None,
-    ) -> Iterable[T_WritableCogniteResource]:
+    ) -> Iterable[T_ResourceResponse]:
         if sum([1 for x in [data_set_external_id, space, parent_ids] if x is not None]) > 1:
             raise ValueError("At most one of data_set_external_id, space, or parent_ids must be set.")
         if parent_ids is not None and not self.parent_resource:
@@ -252,14 +211,17 @@ class ResourceCRUD(
         data_set_external_id: str | None = None,
         space: str | None = None,
         parent_ids: list[Hashable] | None = None,
-    ) -> Iterable[T_WritableCogniteResource]:
+    ) -> Iterable[T_ResourceResponse]:
         raise NotImplementedError
 
-    # The methods below have default implementations that can be overwritten in subclasses
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        return read_parameter_from_init_type_hints(cls.resource_write_cls).as_camel_case()
+    ### These methods can be optionally overwritten in the subclass ###
+    def prerequisite_warning(self) -> str | None:
+        """ "Returns a warning message if there are any prerequisites that must be met before using this CRUD.
+
+        This is used for special resource CRUDs that for example require data models/views to be deployed in CDF
+        to work. For example, the InfieldV1CRUD and the ResourceViewMappingCRUD.
+        """
+        return None
 
     @classmethod
     def get_dependent_items(cls, item: dict) -> "Iterable[tuple[type[ResourceCRUD], Hashable]]":
@@ -292,7 +254,7 @@ class ResourceCRUD(
         return []
 
     @classmethod
-    def get_internal_id(cls, item: T_WritableCogniteResource | dict) -> int:
+    def get_internal_id(cls, item: T_ResourceResponse | dict) -> int:
         raise NotImplementedError(f"{cls.__name__} does not have an internal id.")
 
     @classmethod
@@ -329,18 +291,18 @@ class ResourceCRUD(
         )
         return raw_yaml if isinstance(raw_yaml, list) else [raw_yaml]
 
-    def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> T_WriteClass:
+    def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> T_ResourceRequest:
         """Loads the resource from a dictionary. Can be overwritten in subclasses."""
         return self.resource_write_cls._load(resource)
 
-    def dump_resource(self, resource: T_WritableCogniteResource, local: dict[str, Any] | None = None) -> dict[str, Any]:
+    def dump_resource(self, resource: T_ResourceResponse, local: dict[str, Any] | None = None) -> dict[str, Any]:
         """Dumps the resource to a dictionary that matches the write format.
 
         This is intended to be overwritten in subclasses that require special dumping logic, for example,
         replacing dataSetId with dataSetExternalId.
 
         Args:
-            resource (T_WritableCogniteResource): The resource to dump (typically comes from CDF).
+            resource (T_ResourceResponse): The resource to dump (typically comes from CDF).
             local (dict[str, Any] | None): The local resource. When used in a dump/import command, there is no local
                 resource.
         """
@@ -389,7 +351,7 @@ class ResourceCRUD(
             f"Missing implementation for {type(self).__name__} for {'.'.join(map(str, json_path))}."
         )
 
-    def sensitive_strings(self, item: T_WriteClass) -> Iterable[str]:
+    def sensitive_strings(self, item: T_ResourceRequest) -> Iterable[str]:
         """Returns a list of strings that should be masked when printing.
 
         This is used by the loaders with credentials to mask the credentials secrets. For example, the
@@ -400,39 +362,19 @@ class ResourceCRUD(
 
     # Helper methods
     @classmethod
-    def get_ids(cls, items: Sequence[T_WriteClass | T_WritableCogniteResource | dict]) -> list[T_ID]:
+    def get_ids(cls, items: Sequence[T_ResourceRequest | T_ResourceResponse | dict]) -> list[T_ID]:
         return [cls.get_id(item) for item in items]
 
-    @classmethod
-    def safe_get_write_cls_parameter_spec(cls) -> ParameterSpecSet | None:
-        from sentry_sdk import capture_exception
-
-        api_spec: ParameterSpecSet | None = None
-        try:
-            api_spec = cls.get_write_cls_parameter_spec()
-        except Exception as e:
-            # We don't want to crash the program if we can't get the parameter spec
-            # as we can continue without doing this check. Note that getting the parameter spec
-            # is also fragile as it relies on the type hints in the cognite-sdk which is out of our control.
-            if USE_SENTRY:
-                capture_exception(e)
-            else:
-                raise
-        return api_spec
-
     @classmethod
     def as_str(cls, id: T_ID) -> str:
         if isinstance(id, str):
-            return to_directory_compatible(id)
+            return sanitize_filename(id)
         raise NotImplementedError(
             f"Bug in CogniteToolkit 'as_str' is not implemented for {cls.__name__.removesuffix('Loader')}."
         )
 
 
-class ResourceContainerCRUD(
-    ResourceCRUD[T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList],
-    ABC,
-):
+class ResourceContainerCRUD(ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse], ABC):
     """This is the base class for all resource CRUD' containers.
 
     A resource container CRUD is a resource that contains data. For example, Timeseries contains datapoints, and another
@@ -480,10 +422,3 @@ class DataCRUD(Loader, ABC):
     @abstractmethod
     def upload(self, state: "BuildEnvironment", dry_run: bool) -> Iterable[tuple[str, int]]:
         raise NotImplementedError
-
-    def _find_data_files(self, directory: Path) -> list[Path]:
-        return [
-            path
-            for path in directory.rglob("*")
-            if path.is_file() and path.name not in EXCL_FILES and self.is_supported_file(path)
-        ]

cognite_toolkit/_cdf_tk/cruds/_data_cruds.py

@@ -5,15 +5,18 @@ from typing import TYPE_CHECKING, cast, final
 
 import pandas as pd
 from cognite.client.data_classes import FileMetadataWrite
-from cognite.client.data_classes._base import T_CogniteResourceList, T_WritableCogniteResource, T_WriteClass
 
 from cognite_toolkit._cdf_tk.client.data_classes.extendable_cognite_file import ExtendableCogniteFileApply
 from cognite_toolkit._cdf_tk.client.data_classes.raw import RawTable
 from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING
+from cognite_toolkit._cdf_tk.protocols import (
+    T_ResourceRequest,
+    T_ResourceResponse,
+)
 from cognite_toolkit._cdf_tk.utils import read_yaml_content, safe_read
 from cognite_toolkit._cdf_tk.utils.file import read_csv
 
-from ._base_cruds import T_ID, DataCRUD, ResourceCRUD, T_WritableCogniteResourceList
+from ._base_cruds import T_ID, DataCRUD, ResourceCRUD
 from ._resource_cruds import CogniteFileCRUD, FileMetadataCRUD, RawTableCRUD, TimeSeriesCRUD
 
 if TYPE_CHECKING:
@@ -25,7 +28,6 @@ class DatapointsCRUD(DataCRUD):
     item_name = "datapoints"
     folder_name = "timeseries"
     kind = "Datapoints"
-    filetypes = frozenset({"csv", "parquet"})
     dependencies = frozenset({TimeSeriesCRUD})
     _doc_url = "Time-series/operation/postMultiTimeSeriesDatapoints"
 
@@ -40,7 +42,9 @@ class DatapointsCRUD(DataCRUD):
         resource_directories = state.built_resources[self.folder_name].get_resource_directories(self.folder_name)
 
         for resource_dir in resource_directories:
-            for datafile in self._find_data_files(resource_dir):
+            for datafile in resource_dir.rglob("*"):
+                if not datafile.stem.casefold().endswith(self.kind.casefold()):
+                    continue
                 if datafile.suffix == ".csv":
                     # The replacement is used to ensure that we read exactly the same file on Windows and Linux
                     file_content = datafile.read_bytes().replace(b"\r\n", b"\n").decode("utf-8")
@@ -49,7 +53,7 @@ class DatapointsCRUD(DataCRUD):
                 elif datafile.suffix == ".parquet":
                     data = pd.read_parquet(datafile, engine="pyarrow")
                 else:
-                    raise ValueError(f"Unsupported file type {datafile.suffix} for {datafile.name}")
+                    continue
                 timeseries_ids = list(data.columns)
                 if len(timeseries_ids) == 1:
                     ts_str = timeseries_ids[0]
@@ -84,12 +88,6 @@ class FileCRUD(DataCRUD):
     item_name = "file contents"
     folder_name = "files"
     kind = "File"
-    filetypes = frozenset()
-    exclude_filetype: frozenset[str] = frozenset({})
-    filename_pattern = (
-        # Exclude FileMetadata and CogniteFile
-        r"(?i)^(?!.*(?:FileMetadata|CogniteFile)$).*$"
-    )
     dependencies = frozenset({FileMetadataCRUD, CogniteFileCRUD})
     _doc_url = "Files/operation/initFileUpload"
 
@@ -130,19 +128,16 @@ class FileCRUD(DataCRUD):
     @staticmethod
     def _read_metadata(
         destination: Path,
-        loader: type[
-            ResourceCRUD[
-                T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-            ]
-        ],
+        loader: type[ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse]],
         identifier: T_ID,
-    ) -> T_WriteClass:
+    ) -> T_ResourceRequest:
         built_content = read_yaml_content(safe_read(destination, encoding=BUILD_FOLDER_ENCODING))
         if isinstance(built_content, dict):
-            return loader.resource_write_cls.load(built_content)
+            return loader.resource_write_cls._load(built_content)
         elif isinstance(built_content, list):
+            write_resources = (loader.resource_write_cls._load(content) for content in built_content)
             try:
-                return next(m for m in loader.list_write_cls.load(built_content) if loader.get_id(m) == identifier)
+                return next(m for m in write_resources if loader.get_id(m) == identifier)
             except StopIteration:
                 raise RuntimeError(f"Missing metadata for {destination.as_posix()}")
 
@@ -153,7 +148,6 @@ class FileCRUD(DataCRUD):
 class RawFileCRUD(DataCRUD):
     item_name = "rows"
     folder_name = "raw"
-    filetypes = frozenset({"csv", "parquet"})
     kind = "Raw"
     dependencies = frozenset({RawTableCRUD})
     _doc_url = "Raw/operation/postRows"
@@ -173,7 +167,7 @@ class RawFileCRUD(DataCRUD):
             datafile = next(
                 (
                     resource.source.path.with_suffix(f".{file_type}")
-                    for file_type in self.filetypes
+                    for file_type in ["csv", "parquet"]
                     if (resource.source.path.with_suffix(f".{file_type}").exists())
                 ),
                 None,

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/__init__.py

@@ -13,7 +13,7 @@ from .datamodel import (
     ViewCRUD,
 )
 from .extraction_pipeline import ExtractionPipelineConfigCRUD, ExtractionPipelineCRUD
-from .fieldops import InfieldV1CRUD
+from .fieldops import InFieldCDMLocationConfigCRUD, InFieldLocationConfigCRUD, InfieldV1CRUD
 from .file import CogniteFileCRUD, FileMetadataCRUD
 from .function import FunctionCRUD, FunctionScheduleCRUD
 from .group_scoped import GroupResourceScopedCRUD
@@ -35,6 +35,7 @@ from .robotics import (
     RoboticMapCRUD,
     RoboticsDataPostProcessingCRUD,
 )
+from .streams import StreamCRUD
 from .three_d_model import ThreeDModelCRUD
 from .timeseries import DatapointSubscriptionCRUD, TimeSeriesCRUD
 from .transformation import TransformationCRUD, TransformationNotificationCRUD, TransformationScheduleCRUD
@@ -63,6 +64,8 @@ __all__ = [
     "HostedExtractorJobCRUD",
     "HostedExtractorMappingCRUD",
     "HostedExtractorSourceCRUD",
+    "InFieldCDMLocationConfigCRUD",
+    "InFieldLocationConfigCRUD",
     "InfieldV1CRUD",
     "LabelCRUD",
     "LocationFilterCRUD",
@@ -81,6 +84,7 @@ __all__ = [
     "SequenceCRUD",
     "SequenceRowCRUD",
     "SpaceCRUD",
+    "StreamCRUD",
     "StreamlitCRUD",
     "ThreeDModelCRUD",
     "TimeSeriesCRUD",

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/agent.py

@@ -1,26 +1,22 @@
 from collections.abc import Hashable, Iterable, Sequence
-from functools import lru_cache
 from typing import Any
 
-from cognite.client.data_classes.agents import Agent, AgentList, AgentUpsert, AgentUpsertList
-from cognite.client.data_classes.capabilities import Capability
+from cognite.client.data_classes.agents import Agent, AgentList, AgentUpsert
+from cognite.client.data_classes.capabilities import AgentsAcl, Capability
 from cognite.client.exceptions import CogniteAPIError
 from cognite.client.utils.useful_types import SequenceNotStr
 
-from cognite_toolkit._cdf_tk._parameters.constants import ANY_INT, ANYTHING
-from cognite_toolkit._cdf_tk._parameters.data_classes import ParameterSpec, ParameterSpecSet
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
+from cognite_toolkit._cdf_tk.resource_classes import AgentYAML
 from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_hashable, diff_list_identifiable
 
 
-class AgentCRUD(ResourceCRUD[str, AgentUpsert, Agent, AgentUpsertList, AgentList]):
+class AgentCRUD(ResourceCRUD[str, AgentUpsert, Agent]):
     folder_name = "agents"
-    filename_pattern = r".*\.Agent$"  # Matches all yaml files whose stem ends with '.Agent'.
     resource_cls = Agent
     resource_write_cls = AgentUpsert
-    list_cls = AgentList
-    list_write_cls = AgentUpsertList
     kind = "Agent"
+    yaml_cls = AgentYAML
     _doc_base_url = ""
     _doc_url = "https://api-docs.cognite.com/20230101-beta/tag/Agents/operation/main_ai_agents_post/"
 
@@ -38,30 +34,20 @@ class AgentCRUD(ResourceCRUD[str, AgentUpsert, Agent, AgentUpsertList, AgentList
     def get_required_capability(
         cls, items: Sequence[AgentUpsert] | None, read_only: bool
     ) -> Capability | list[Capability]:
-        return []
+        if not items and items is not None:
+            return []
 
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        spec = super().get_write_cls_parameter_spec()
-        # Tool configuration is a dict where the accepted keys depend on the tool type.
-        spec.add(
-            ParameterSpec(
-                ("tools", ANY_INT, "configuration", ANYTHING),
-                frozenset({"dict"}),
-                is_required=False,
-                _is_nullable=False,
-            )
-        )
-        return spec
+        actions = [AgentsAcl.Action.READ] if read_only else [AgentsAcl.Action.READ, AgentsAcl.Action.WRITE]
+
+        return AgentsAcl(actions, AgentsAcl.Scope.All())
 
-    def create(self, items: AgentUpsertList) -> AgentList:
+    def create(self, items: Sequence[AgentUpsert]) -> AgentList:
         return self.client.agents.upsert(items)
 
     def retrieve(self, ids: SequenceNotStr[str]) -> AgentList:
         return self.client.agents.retrieve(ids, ignore_unknown_ids=True)
 
-    def update(self, items: AgentUpsertList) -> AgentList:
+    def update(self, items: Sequence[AgentUpsert]) -> AgentList:
         return self.client.agents.upsert(items)
 
     def delete(self, ids: SequenceNotStr[str]) -> int: