cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/{loaders/_resource_loaders → cruds/_resource_cruds}/hosted_extractors.py

@@ -1,7 +1,4 @@
-from __future__ import annotations
-
 from collections.abc import Hashable, Iterable, Sequence
-from functools import lru_cache
 from pathlib import Path
 from typing import Any
 
@@ -11,21 +8,17 @@ from cognite.client.data_classes.hosted_extractors import (
     Destination,
     DestinationList,
     DestinationWrite,
-    DestinationWriteList,
     EventHubSourceWrite,
     Job,
     JobList,
     JobWrite,
-    JobWriteList,
     KafkaSourceWrite,
     Mapping,
     MappingList,
     MappingWrite,
-    MappingWriteList,
     Source,
     SourceList,
     SourceWrite,
-    SourceWriteList,
 )
 from cognite.client.data_classes.hosted_extractors.sources import (
     AuthenticationWrite,
@@ -37,23 +30,26 @@ from cognite.client.data_classes.hosted_extractors.sources import (
 from cognite.client.utils.useful_types import SequenceNotStr
 from rich.console import Console
 
-from cognite_toolkit._cdf_tk._parameters import ANYTHING, ParameterSpec, ParameterSpecSet
 from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
 from cognite_toolkit._cdf_tk.exceptions import ToolkitNotSupported
-from cognite_toolkit._cdf_tk.loaders._base_loaders import ResourceLoader
+from cognite_toolkit._cdf_tk.resource_classes import (
+    HostedExtractorDestinationYAML,
+    HostedExtractorJobYAML,
+    HostedExtractorMappingYAML,
+    HostedExtractorSourceYAML,
+)
 from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning
 
-from .data_organization_loaders import DataSetsLoader
+from .data_organization import DataSetsCRUD
 
 
-class HostedExtractorSourceLoader(ResourceLoader[str, SourceWrite, Source, SourceWriteList, SourceList]):
+class HostedExtractorSourceCRUD(ResourceCRUD[str, SourceWrite, Source]):
     folder_name = "hosted_extractors"
-    filename_pattern = r".*\.Source$"  # Matches all yaml files whose stem ends with '.Source'.
     resource_cls = Source
     resource_write_cls = SourceWrite
-    list_cls = SourceList
-    list_write_cls = SourceWriteList
     kind = "Source"
+    yaml_cls = HostedExtractorSourceYAML
     _doc_base_url = "https://api-docs.cognite.com/20230101-alpha/tag/"
     _doc_url = "Sources/operation/create_sources"
     _SupportedSources = (_MQTTSourceWrite, KafkaSourceWrite, RestSourceWrite, EventHubSourceWrite)
@@ -90,13 +86,13 @@ class HostedExtractorSourceLoader(ResourceLoader[str, SourceWrite, Source, Sourc
             HostedExtractorsAcl.Scope.All(),
         )
 
-    def create(self, items: SourceWriteList) -> SourceList:
+    def create(self, items: Sequence[SourceWrite]) -> SourceList:
         return self.client.hosted_extractors.sources.create(items)
 
     def retrieve(self, ids: SequenceNotStr[str]) -> SourceList:
         return self.client.hosted_extractors.sources.retrieve(external_ids=ids, ignore_unknown_ids=True)
 
-    def update(self, items: SourceWriteList) -> SourceList:
+    def update(self, items: Sequence[SourceWrite]) -> SourceList:
         return self.client.hosted_extractors.sources.update(items, mode="replace")
 
     def delete(self, ids: SequenceNotStr[str]) -> int:
@@ -111,14 +107,6 @@ class HostedExtractorSourceLoader(ResourceLoader[str, SourceWrite, Source, Sourc
     ) -> Iterable[Source]:
         return iter(self.client.hosted_extractors.sources)
 
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        # parameterspec is highly dependent on type of source, so we accept any parameter
-        return ParameterSpecSet(
-            [ParameterSpec((ANYTHING,), frozenset({"dict"}), is_required=False, _is_nullable=False)]
-        )
-
     def dump_resource(self, resource: Source, local: dict[str, Any] | None = None) -> dict[str, Any]:
         HighSeverityWarning(
             "Sources will always be considered different, and thus will always be redeployed."
@@ -156,19 +144,15 @@ class HostedExtractorSourceLoader(ResourceLoader[str, SourceWrite, Source, Sourc
             yield auth.client_secret
 
 
-class HostedExtractorDestinationLoader(
-    ResourceLoader[str, DestinationWrite, Destination, DestinationWriteList, DestinationList]
-):
+class HostedExtractorDestinationCRUD(ResourceCRUD[str, DestinationWrite, Destination]):
     folder_name = "hosted_extractors"
-    filename_pattern = r".*\.Destination$"  # Matches all yaml files whose stem ends with '.Destination'.
     resource_cls = Destination
     resource_write_cls = DestinationWrite
-    list_cls = DestinationList
-    list_write_cls = DestinationWriteList
-    dependencies = frozenset({DataSetsLoader})
+    dependencies = frozenset({DataSetsCRUD})
     kind = "Destination"
     _doc_base_url = "https://api-docs.cognite.com/20230101-alpha/tag/"
     _doc_url = "Destinations/operation/create_destinations"
+    yaml_cls = HostedExtractorDestinationYAML
 
     def __init__(self, client: ToolkitClient, build_dir: Path | None, console: Console | None = None):
         super().__init__(client, build_dir, console)
@@ -206,13 +190,13 @@ class HostedExtractorDestinationLoader(
             HostedExtractorsAcl.Scope.All(),
         )
 
-    def create(self, items: DestinationWriteList) -> DestinationList:
+    def create(self, items: Sequence[DestinationWrite]) -> DestinationList:
         return self.client.hosted_extractors.destinations.create(items)
 
     def retrieve(self, ids: SequenceNotStr[str]) -> DestinationList:
         return self.client.hosted_extractors.destinations.retrieve(external_ids=ids, ignore_unknown_ids=True)
 
-    def update(self, items: DestinationWriteList) -> DestinationList:
+    def update(self, items: Sequence[DestinationWrite]) -> DestinationList:
         return self.client.hosted_extractors.destinations.update(items, mode="replace")
 
     def delete(self, ids: SequenceNotStr[str]) -> int:
@@ -247,24 +231,9 @@ class HostedExtractorDestinationLoader(
         return self.dump_id(resource.external_id)
 
     @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        spec = super().get_write_cls_parameter_spec()
-        # Handled by Toolkit
-
-        spec.add(ParameterSpec(("credentials", "clientId"), frozenset({"str"}), is_required=False, _is_nullable=True))
-        spec.add(
-            ParameterSpec(("credentials", "clientSecret"), frozenset({"str"}), is_required=False, _is_nullable=True)
-        )
-
-        spec.discard(ParameterSpec(("targetDataSetId",), frozenset({"int"}), is_required=False, _is_nullable=True))
-        spec.add(ParameterSpec(("targetDataSetExternalId",), frozenset({"str"}), is_required=False, _is_nullable=True))
-        return spec
-
-    @classmethod
-    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader], Hashable]]:
+    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
         if "targetDataSetId" in item:
-            yield DataSetsLoader, item["targetDataSetId"]
+            yield DataSetsCRUD, item["targetDataSetId"]
 
     def sensitive_strings(self, item: DestinationWrite) -> Iterable[str]:
         yield item.credentials.nonce
@@ -273,15 +242,13 @@ class HostedExtractorDestinationLoader(
             yield self._authentication_by_id[id_].client_secret
 
 
-class HostedExtractorJobLoader(ResourceLoader[str, JobWrite, Job, JobWriteList, JobList]):
+class HostedExtractorJobCRUD(ResourceCRUD[str, JobWrite, Job]):
     folder_name = "hosted_extractors"
-    filename_pattern = r".*\.Job$"  # Matches all yaml files whose stem ends with '.Job'.
     resource_cls = Job
     resource_write_cls = JobWrite
-    list_cls = JobList
-    list_write_cls = JobWriteList
-    dependencies = frozenset({HostedExtractorSourceLoader, HostedExtractorDestinationLoader})
+    dependencies = frozenset({HostedExtractorSourceCRUD, HostedExtractorDestinationCRUD})
     kind = "Job"
+    yaml_cls = HostedExtractorJobYAML
     _doc_base_url = "https://api-docs.cognite.com/20230101-alpha/tag/"
     _doc_url = "Jobs/operation/create_jobs"
 
@@ -324,13 +291,13 @@ class HostedExtractorJobLoader(ResourceLoader[str, JobWrite, Job, JobWriteList,
             dumped.pop("config", None)
         return dumped
 
-    def create(self, items: JobWriteList) -> JobList:
+    def create(self, items: Sequence[JobWrite]) -> JobList:
         return self.client.hosted_extractors.jobs.create(items)
 
     def retrieve(self, ids: SequenceNotStr[str]) -> JobList:
         return self.client.hosted_extractors.jobs.retrieve(external_ids=ids, ignore_unknown_ids=True)
 
-    def update(self, items: JobWriteList) -> JobList:
+    def update(self, items: Sequence[JobWrite]) -> JobList:
         return self.client.hosted_extractors.jobs.update(items, mode="replace")
 
     def delete(self, ids: SequenceNotStr[str]) -> int:
@@ -346,52 +313,23 @@ class HostedExtractorJobLoader(ResourceLoader[str, JobWrite, Job, JobWriteList,
         return iter(self.client.hosted_extractors.jobs)
 
     @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        spec = super().get_write_cls_parameter_spec()
-        # Used by the SDK to determine the class to load
-        spec.add(
-            ParameterSpec(
-                (
-                    "format",
-                    "type",
-                ),
-                frozenset({"str"}),
-                is_required=True,
-                _is_nullable=False,
-            )
-        )
-        spec.add(
-            ParameterSpec(
-                ("config", "incrementalLoad", "type"), frozenset({"str"}), is_required=True, _is_nullable=False
-            )
-        )
-        spec.add(
-            ParameterSpec(("config", "pagination", "type"), frozenset({"str"}), is_required=True, _is_nullable=False)
-        )
-
-        return spec
-
-    @classmethod
-    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader], Hashable]]:
+    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
         if "sourceId" in item:
-            yield HostedExtractorSourceLoader, item["sourceId"]
+            yield HostedExtractorSourceCRUD, item["sourceId"]
         if "destinationId" in item:
-            yield HostedExtractorDestinationLoader, item["destinationId"]
+            yield HostedExtractorDestinationCRUD, item["destinationId"]
 
 
-class HostedExtractorMappingLoader(ResourceLoader[str, MappingWrite, Mapping, MappingWriteList, MappingList]):
+class HostedExtractorMappingCRUD(ResourceCRUD[str, MappingWrite, Mapping]):
     folder_name = "hosted_extractors"
-    filename_pattern = r".*\.Mapping$"  # Matches all yaml files whose stem ends with '.Mapping'.
     resource_cls = Mapping
     resource_write_cls = MappingWrite
-    list_cls = MappingList
-    list_write_cls = MappingWriteList
     # This is not an explicit dependency, however, adding it here as mapping will should be deployed after source.
-    dependencies = frozenset({HostedExtractorSourceLoader})
+    dependencies = frozenset({HostedExtractorSourceCRUD})
     kind = "Mapping"
     _doc_base_url = "https://api-docs.cognite.com/20230101-alpha/tag/"
     _doc_url = "Mappings/operation/create_mappings"
+    yaml_cls = HostedExtractorMappingYAML
 
     @property
     def display_name(self) -> str:
@@ -425,13 +363,13 @@ class HostedExtractorMappingLoader(ResourceLoader[str, MappingWrite, Mapping, Ma
             HostedExtractorsAcl.Scope.All(),
         )
 
-    def create(self, items: MappingWriteList) -> MappingList:
+    def create(self, items: Sequence[MappingWrite]) -> MappingList:
         return self.client.hosted_extractors.mappings.create(items)
 
     def retrieve(self, ids: SequenceNotStr[str]) -> MappingList:
         return self.client.hosted_extractors.mappings.retrieve(external_ids=ids, ignore_unknown_ids=True)
 
-    def update(self, items: MappingWriteList) -> MappingList:
+    def update(self, items: Sequence[MappingWrite]) -> MappingList:
         return self.client.hosted_extractors.mappings.update(items)
 
     def delete(self, ids: SequenceNotStr[str]) -> int:
@@ -445,11 +383,3 @@ class HostedExtractorMappingLoader(ResourceLoader[str, MappingWrite, Mapping, Ma
         parent_ids: list[Hashable] | None = None,
     ) -> Iterable[Mapping]:
         return iter(self.client.hosted_extractors.mappings)
-
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        spec = super().get_write_cls_parameter_spec()
-        # Used by the SDK to determine the class to load
-        spec.add(ParameterSpec(("input", "type"), frozenset({"str"}), is_required=True, _is_nullable=False))
-        return spec

cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/industrial_tool_loaders.py → cruds/_resource_cruds/industrial_tool.py}

@@ -13,38 +13,35 @@ from cognite.client.utils.useful_types import SequenceNotStr
 from packaging.requirements import Requirement
 from rich.console import Console
 
-from cognite_toolkit._cdf_tk._parameters import ParameterSpec, ParameterSpecSet
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.streamlit_ import (
     Streamlit,
     StreamlitList,
     StreamlitWrite,
-    StreamlitWriteList,
 )
+from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
 from cognite_toolkit._cdf_tk.exceptions import ToolkitNotADirectoryError, ToolkitRequiredValueError
-from cognite_toolkit._cdf_tk.loaders._base_loaders import ResourceLoader
+from cognite_toolkit._cdf_tk.resource_classes import StreamlitYAML
 from cognite_toolkit._cdf_tk.utils import (
     load_yaml_inject_variables,
     safe_read,
 )
-from cognite_toolkit._cdf_tk.utils.hashing import calculate_str_or_file_hash
+from cognite_toolkit._cdf_tk.utils.hashing import calculate_hash
 
-from .auth_loaders import GroupAllScopedLoader
-from .data_organization_loaders import DataSetsLoader
+from .auth import GroupAllScopedCRUD
+from .data_organization import DataSetsCRUD
 
 
 @final
-class StreamlitLoader(ResourceLoader[str, StreamlitWrite, Streamlit, StreamlitWriteList, StreamlitList]):
+class StreamlitCRUD(ResourceCRUD[str, StreamlitWrite, Streamlit]):
     folder_name = "streamlit"
-    filename_pattern = r".*streamlit$"
     resource_cls = Streamlit
     resource_write_cls = StreamlitWrite
-    list_cls = StreamlitList
-    list_write_cls = StreamlitWriteList
     kind = "Streamlit"
-    dependencies = frozenset({DataSetsLoader, GroupAllScopedLoader})
+    dependencies = frozenset({DataSetsCRUD, GroupAllScopedCRUD})
     _doc_url = "Files/operation/initFileUpload"
     _metadata_hash_key = "cdf-toolkit-app-hash"
+    yaml_cls = StreamlitYAML
 
     @property
     def display_name(self) -> str:
@@ -72,7 +69,7 @@ class StreamlitLoader(ResourceLoader[str, StreamlitWrite, Streamlit, StreamlitWr
             if data_set_ids := {item.data_set_id for item in items if item.data_set_id}:
                 scope = FilesAcl.Scope.DataSet(list(data_set_ids))
 
-        return FilesAcl(actions, scope)  # type: ignore[arg-type]
+        return FilesAcl(actions, scope)
 
     @classmethod
     def get_id(cls, item: Streamlit | StreamlitWrite | dict) -> str:
@@ -87,9 +84,9 @@ class StreamlitLoader(ResourceLoader[str, StreamlitWrite, Streamlit, StreamlitWr
         return {"externalId": id}
 
     @classmethod
-    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader], Hashable]]:
+    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
         if "dataSetExternalId" in item:
-            yield DataSetsLoader, item["dataSetExternalId"]
+            yield DataSetsCRUD, item["dataSetExternalId"]
 
     def load_resource_file(
         self, filepath: Path, environment_variables: dict[str, str | None] | None = None
@@ -104,7 +101,7 @@ class StreamlitLoader(ResourceLoader[str, StreamlitWrite, Streamlit, StreamlitWr
             item_id = self.get_id(item)
             self._source_file_by_external_id[item_id] = filepath
             content = self._as_json_string(item_id, item["entrypoint"])
-            item["cogniteToolkitAppHash"] = calculate_str_or_file_hash(content, shorten=True)
+            item["cogniteToolkitAppHash"] = calculate_hash(content, shorten=True)
         return raw_list
 
     def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> StreamlitWrite:
@@ -151,12 +148,12 @@ class StreamlitLoader(ResourceLoader[str, StreamlitWrite, Streamlit, StreamlitWr
     def _missing_recommended_requirements(requirements: list[str]) -> list[str]:
         missing = []
         user_requirements = {Requirement(req).name for req in requirements}
-        for recommended in StreamlitLoader.recommended_packages():
+        for recommended in StreamlitCRUD.recommended_packages():
             if recommended.name not in user_requirements:
                 missing.append(recommended.name)
         return missing
 
-    def create(self, items: StreamlitWriteList) -> StreamlitList:
+    def create(self, items: Sequence[StreamlitWrite]) -> StreamlitList:
         created = StreamlitList([])
         for item in items:
             content = self._as_json_string(item.external_id, item.entrypoint)
@@ -171,7 +168,7 @@ class StreamlitLoader(ResourceLoader[str, StreamlitWrite, Streamlit, StreamlitWr
         files = self.client.files.retrieve_multiple(external_ids=ids, ignore_unknown_ids=True)
         return StreamlitList([Streamlit.from_file(file) for file in files])
 
-    def update(self, items: StreamlitWriteList) -> StreamlitList:
+    def update(self, items: Sequence[StreamlitWrite]) -> StreamlitList:
         files = []
         for item in items:
             content = self._as_json_string(item.external_id, item.entrypoint)
@@ -195,12 +192,3 @@ class StreamlitLoader(ResourceLoader[str, StreamlitWrite, Streamlit, StreamlitWr
         for file in self.client.files:
             if file.directory == "/streamlit-apps/":
                 yield Streamlit.from_file(file)
-
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        spec = super().get_write_cls_parameter_spec()
-        # Added by toolkit
-        spec.add(ParameterSpec(("dataSetExternalId",), frozenset({"str"}), is_required=False, _is_nullable=True))
-        spec.discard(ParameterSpec(("dataSetId",), frozenset({"int"}), is_required=False, _is_nullable=True))
-        return spec