cognite-toolkit 0.5.97__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/{loaders/_resource_loaders/datamodel_loaders.py → cruds/_resource_cruds/datamodel.py}

@@ -18,7 +18,6 @@ import sys
 import time
 from collections import defaultdict
 from collections.abc import Hashable, Iterable, Sequence
-from functools import lru_cache
 from graphlib import CycleError, TopologicalSorter
 from pathlib import Path
 from time import sleep
@@ -35,29 +34,26 @@ from cognite.client.data_classes.capabilities import (
 from cognite.client.data_classes.data_modeling import (
     Container,
     ContainerApply,
-    ContainerApplyList,
     ContainerList,
     DataModel,
     DataModelApply,
-    DataModelApplyList,
     DataModelList,
+    DirectRelation,
     Edge,
     EdgeApply,
-    EdgeApplyList,
     EdgeApplyResultList,
     EdgeList,
+    MappedProperty,
     Node,
     NodeApply,
-    NodeApplyList,
     NodeApplyResultList,
     NodeList,
+    RequiresConstraint,
     Space,
     SpaceApply,
-    SpaceApplyList,
     SpaceList,
     View,
     ViewApply,
-    ViewApplyList,
     ViewList,
 )
 from cognite.client.data_classes.data_modeling.graphql import DMLApplyResult
@@ -68,7 +64,6 @@ from cognite.client.data_classes.data_modeling.ids import (
     NodeId,
     ViewId,
 )
-from cognite.client.data_classes.data_modeling.views import ReverseDirectRelationApply
 from cognite.client.exceptions import CogniteAPIError
 from cognite.client.utils.useful_types import SequenceNotStr
 from rich import print
@@ -76,21 +71,28 @@ from rich.console import Console
 from rich.markup import escape
 from rich.panel import Panel
 
-from cognite_toolkit._cdf_tk._parameters import ANY_INT, ANY_STR, ANYTHING, ParameterSpec, ParameterSpecSet
+from cognite_toolkit._cdf_tk import constants
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.graphql_data_models import (
     GraphQLDataModel,
     GraphQLDataModelList,
     GraphQLDataModelWrite,
-    GraphQLDataModelWriteList,
 )
 from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING, HAS_DATA_FILTER_LIMIT
+from cognite_toolkit._cdf_tk.cruds._base_cruds import (
+    ResourceContainerCRUD,
+    ResourceCRUD,
+)
 from cognite_toolkit._cdf_tk.exceptions import GraphQLParseError, ToolkitCycleError, ToolkitFileNotFoundError
-from cognite_toolkit._cdf_tk.loaders._base_loaders import (
-    ResourceContainerLoader,
-    ResourceLoader,
+from cognite_toolkit._cdf_tk.resource_classes import (
+    ContainerYAML,
+    DataModelYAML,
+    EdgeYAML,
+    GraphQLDataModelYAML,
+    NodeYAML,
+    SpaceYAML,
+    ViewYAML,
 )
-from cognite_toolkit._cdf_tk.resource_classes import SpaceYAML
 from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, LowSeverityWarning, MediumSeverityWarning
 from cognite_toolkit._cdf_tk.utils import (
     GraphQLParser,
@@ -99,28 +101,24 @@ from cognite_toolkit._cdf_tk.utils import (
     load_yaml_inject_variables,
     quote_int_value_by_key_in_yaml,
     safe_read,
+    sanitize_filename,
     to_diff,
-    to_directory_compatible,
 )
 from cognite_toolkit._cdf_tk.utils.cdf import iterate_instances
 from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_identifiable, dm_identifier
-from cognite_toolkit._cdf_tk.utils.tarjan import tarjan
 
-from .auth_loaders import GroupAllScopedLoader
+from .auth import GroupAllScopedCRUD
 
 
 @final
-class SpaceLoader(ResourceContainerLoader[str, SpaceApply, Space, SpaceApplyList, SpaceList]):
+class SpaceCRUD(ResourceContainerCRUD[str, SpaceApply, Space]):
     item_name = "nodes and edges"
-    folder_name = "data_models"
-    filename_pattern = r"^.*space$"
+    folder_name = "data_modeling"
     resource_cls = Space
     resource_write_cls = SpaceApply
-    list_write_cls = SpaceApplyList
-    list_cls = SpaceList
     kind = "Space"
     yaml_cls = SpaceYAML
-    dependencies = frozenset({GroupAllScopedLoader})
+    dependencies = frozenset({GroupAllScopedCRUD})
     _doc_url = "Spaces/operation/ApplySpaces"
     delete_recreate_limit_seconds: int = 10
 
@@ -242,19 +240,26 @@ class SpaceLoader(ResourceContainerLoader[str, SpaceApply, Space, SpaceApplyList
             yield instances.as_ids()
 
 
-class ContainerLoader(
-    ResourceContainerLoader[ContainerId, ContainerApply, Container, ContainerApplyList, ContainerList]
-):
+class ContainerCRUD(ResourceContainerCRUD[ContainerId, ContainerApply, Container]):
     item_name = "nodes and edges"
-    folder_name = "data_models"
-    filename_pattern = r"^.*container$"
+    folder_name = "data_modeling"
     resource_cls = Container
     resource_write_cls = ContainerApply
-    list_cls = ContainerList
-    list_write_cls = ContainerApplyList
     kind = "Container"
-    dependencies = frozenset({SpaceLoader})
+    dependencies = frozenset({SpaceCRUD})
+    yaml_cls = ContainerYAML
     _doc_url = "Containers/operation/ApplyContainers"
+    sub_folder_name = "containers"
+
+    def __init__(
+        self,
+        client: ToolkitClient,
+        build_dir: Path | None,
+        console: Console | None = None,
+        topological_sort_implements: bool = False,
+    ) -> None:
+        super().__init__(client, build_dir, console)
+        self._container_by_id: dict[ContainerId, Container] = {}
 
     @property
     def display_name(self) -> str:
@@ -275,7 +280,7 @@ class ContainerLoader(
             else DataModelsAcl.Scope.All()
         )
 
-        return DataModelsAcl(actions, scope)  # type: ignore[arg-type]
+        return DataModelsAcl(actions, scope)
 
     @classmethod
     def get_id(cls, item: ContainerApply | Container | dict) -> ContainerId:
@@ -291,9 +296,9 @@ class ContainerLoader(
         return id.dump(include_type=False)
 
     @classmethod
-    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader], Hashable]]:
+    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
         if "space" in item:
-            yield SpaceLoader, item["space"]
+            yield SpaceCRUD, item["space"]
         # Note that we are very careful in the code below to not raise an exception if the
         # item is not properly formed. If that is the case, an appropriate warning will be given elsewhere.
         for prop in item.get("properties", {}).values():
@@ -305,12 +310,13 @@ class ContainerLoader(
                     container = prop_type["container"]
                     if "space" in container and "externalId" in container and container.get("type") == "container":
                         yield (
-                            ContainerLoader,
+                            ContainerCRUD,
                             ContainerId(space=container["space"], external_id=container["externalId"]),
                         )
 
     def dump_resource(self, resource: Container, local: dict[str, Any] | None = None) -> dict[str, Any]:
         dumped = resource.as_write().dump()
+        has_local = local is not None
         local = local or {}
         for key in ["constraints", "indexes"]:
             if not dumped.get(key) and key not in local:
@@ -322,15 +328,16 @@ class ContainerLoader(
                 continue
             local_prop = local_prop_by_id[prop_id]
             for key, default in [("immutable", False), ("autoIncrement", False), ("nullable", True)]:
-                if cdf_prop.get(key) is default and key not in local_prop:
+                if has_local and cdf_prop.get(key) is default and key not in local_prop:
                     cdf_prop.pop(key, None)
             cdf_type = cdf_prop.get("type", {})
             local_type = local_prop.get("type", {})
             for key, type_default in [("list", False), ("collation", "ucs_basic")]:
-                if cdf_type.get(key) == type_default and key not in local_type:
+                if has_local and cdf_type.get(key) == type_default and key not in local_type:
                     cdf_type.pop(key, None)
-        if "usedFor" not in local:
-            dumped.pop("usedFor", None)
+            if has_local and "usedFor" not in local and dumped.get("usedFor") == "node":
+                # Only drop if set to default by server.
+                dumped.pop("usedFor", None)
         return dumped
 
     def create(self, items: Sequence[ContainerApply]) -> ContainerList:
@@ -422,78 +429,89 @@ class ContainerLoader(
             ):
                 yield instances.as_ids()
 
+    def _lookup_containers(self, container_ids: Sequence[ContainerId]) -> dict[ContainerId, Container]:
+        ids_to_lookup = [container_id for container_id in container_ids if container_id not in self._container_by_id]
+        if ids_to_lookup:
+            retrieved_containers = self.client.data_modeling.containers.retrieve(ids_to_lookup)
+            for container in retrieved_containers:
+                self._container_by_id[container.as_id()] = container
+        if missing_container_ids := set(container_ids) - set(self._container_by_id.keys()):
+            MediumSeverityWarning(
+                f"Containers {missing_container_ids} not found or you don't have permission to access them."
+            ).print_warning(console=self.console)
+        return {
+            container_id: self._container_by_id[container_id]
+            for container_id in container_ids
+            if container_id in self._container_by_id
+        }
+
+    def _find_direct_container_dependencies(
+        self, container_ids: Sequence[ContainerId]
+    ) -> dict[ContainerId, set[ContainerId]]:
+        containers_by_id = self._lookup_containers(container_ids)
+        container_dependencies: dict[ContainerId, set[ContainerId]] = defaultdict(set)
+        for container_id, container in containers_by_id.items():
+            for constraint in container.constraints.values():
+                if not isinstance(constraint, RequiresConstraint):
+                    continue
+                container_dependencies[container_id].add(constraint.require)
+            for property in container.properties.values():
+                if not isinstance(property.type, DirectRelation) or property.type.container is None:
+                    continue
+                container_dependencies[container_id].add(property.type.container)
+        return container_dependencies
+
+    def _propagate_indirect_container_dependencies(
+        self, container_dependencies_by_id: dict[ContainerId, set[ContainerId]], dependants: Sequence[ContainerId]
+    ) -> dict[ContainerId, set[ContainerId]]:
+        """Propagate indirect container dependencies using a recursive approach.
+
+        Args:
+            container_dependencies_by_id: Mapping of container IDs to their direct dependencies
+            dependants: Chain of dependant containers to propagate dependencies to
+
+        Returns:
+            Updated dictionary mapping each container ID to all its direct and indirect dependencies
+        """
+        current_container_id = dependants[0]
+        dependencies_to_propagate: set[ContainerId] = set()
+        for container_dependency in container_dependencies_by_id[current_container_id]:
+            if container_dependency in container_dependencies_by_id:
+                # If already processed, propagate its dependencies to current container instead of revisiting it
+                dependencies_to_propagate.update(container_dependencies_by_id[container_dependency])
+                continue
+            self._propagate_indirect_container_dependencies(
+                container_dependencies_by_id, [container_dependency, *dependants]
+            )
+        container_dependencies_by_id[current_container_id].update(dependencies_to_propagate)
+        return container_dependencies_by_id
+
+    def _find_direct_and_indirect_container_dependencies(
+        self, container_ids: Sequence[ContainerId]
+    ) -> dict[ContainerId, set[ContainerId]]:
+        container_dependencies_by_id = self._find_direct_container_dependencies(container_ids)
+        for container_id in list(container_dependencies_by_id.keys()):
+            self._propagate_indirect_container_dependencies(container_dependencies_by_id, [container_id])
+        return container_dependencies_by_id
+
     @staticmethod
     def _chunker(seq: Sequence, size: int) -> Iterable[Sequence]:
         return (seq[pos : pos + size] for pos in range(0, len(seq), size))
 
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        output = super().get_write_cls_parameter_spec()
-        # In the SDK this is called isList, while in the API it is called list.
-        output.discard(
-            ParameterSpec(
-                ("properties", ANY_STR, "type", "isList"), frozenset({"bool"}), is_required=True, _is_nullable=False
-            )
-        )
-        output.add(
-            ParameterSpec(
-                ("properties", ANY_STR, "type", "list"), frozenset({"bool"}), is_required=True, _is_nullable=False
-            )
-        )
-        # The parameters below are used by the SDK to load the correct class, and ase thus not part of the init
-        # that the spec is created from, so we need to add them manually.
-        output.update(
-            ParameterSpecSet(
-                {
-                    ParameterSpec(
-                        ("properties", ANY_STR, "type", "type"),
-                        frozenset({"str"}),
-                        is_required=True,
-                        _is_nullable=False,
-                    ),
-                    ParameterSpec(
-                        # direct relations with constraint
-                        ("properties", ANY_STR, "type", "container", "type"),
-                        frozenset({"str"}),
-                        is_required=True,
-                        _is_nullable=False,
-                    ),
-                    ParameterSpec(
-                        ("constraints", ANY_STR, "constraintType"),
-                        frozenset({"str"}),
-                        is_required=True,
-                        _is_nullable=False,
-                    ),
-                    ParameterSpec(
-                        ("constraints", ANY_STR, "require", "type"),
-                        frozenset({"str"}),
-                        is_required=True,
-                        _is_nullable=False,
-                    ),
-                    ParameterSpec(
-                        ("indexes", ANY_STR, "indexType"), frozenset({"str"}), is_required=True, _is_nullable=False
-                    ),
-                }
-            )
-        )
-        return output
-
     @classmethod
     def as_str(cls, id: ContainerId) -> str:
-        return to_directory_compatible(f"{id.space}_{id.external_id}")
+        return sanitize_filename(f"{id.space}_{id.external_id}")
 
 
-class ViewLoader(ResourceLoader[ViewId, ViewApply, View, ViewApplyList, ViewList]):
-    folder_name = "data_models"
-    filename_pattern = r"^.*view$"
+class ViewCRUD(ResourceCRUD[ViewId, ViewApply, View]):
+    folder_name = "data_modeling"
     resource_cls = View
     resource_write_cls = ViewApply
-    list_cls = ViewList
-    list_write_cls = ViewApplyList
     kind = "View"
-    dependencies = frozenset({SpaceLoader, ContainerLoader})
+    dependencies = frozenset({SpaceCRUD, ContainerCRUD})
+    yaml_cls = ViewYAML
     _doc_url = "Views/operation/ApplyViews"
+    sub_folder_name = "views"
 
     def __init__(
         self,
@@ -525,7 +543,7 @@ class ViewLoader(ResourceLoader[ViewId, ViewApply, View, ViewApplyList, ViewList
             else DataModelsAcl.Scope.All()
         )
 
-        return DataModelsAcl(actions, scope)  # type: ignore[arg-type]
+        return DataModelsAcl(actions, scope)
 
     @classmethod
     def get_id(cls, item: ViewApply | View | dict) -> ViewId:
@@ -542,33 +560,33 @@ class ViewLoader(ResourceLoader[ViewId, ViewApply, View, ViewApplyList, ViewList
         return id.dump(include_type=False)
 
     @classmethod
-    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader], Hashable]]:
+    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
         if "space" in item:
-            yield SpaceLoader, item["space"]
+            yield SpaceCRUD, item["space"]
         if isinstance(implements := item.get("implements", []), list):
             for parent in implements:
                 if not isinstance(parent, dict):
                     continue
                 if parent.get("type") == "view" and in_dict(["space", "externalId", "version"], parent):
                     yield (
-                        ViewLoader,
+                        ViewCRUD,
                         ViewId(parent["space"], parent["externalId"], str(v) if (v := parent.get("version")) else None),
                     )
         for prop in item.get("properties", {}).values():
             if (container := prop.get("container", {})) and container.get("type") == "container":
                 if in_dict(("space", "externalId"), container):
-                    yield ContainerLoader, ContainerId(container["space"], container["externalId"])
+                    yield ContainerCRUD, ContainerId(container["space"], container["externalId"])
             for key, dct_ in [("source", prop), ("edgeSource", prop), ("source", prop.get("through", {}))]:
                 if source := dct_.get(key, {}):
                     if source.get("type") == "view" and in_dict(("space", "externalId", "version"), source):
                         yield (
-                            ViewLoader,
+                            ViewCRUD,
                             ViewId(
                                 source["space"], source["externalId"], str(v) if (v := source.get("version")) else None
                             ),
                         )
                     elif source.get("type") == "container" and in_dict(("space", "externalId"), source):
-                        yield ContainerLoader, ContainerId(source["space"], source["externalId"])
+                        yield ContainerCRUD, ContainerId(source["space"], source["externalId"])
 
     def safe_read(self, filepath: Path | str) -> str:
         # The version is a string, but the user often writes it as an int.
@@ -605,7 +623,9 @@ class ViewLoader(ResourceLoader[ViewId, ViewApply, View, ViewApplyList, ViewList
             # We sort the implements in topological order to ensure that the child view get the order grandparent,
             # parent, such that the parent's source is used.
             try:
-                dumped["implements"] = [view_id.dump() for view_id in self.topological_sort(resource.implements)]
+                dumped["implements"] = [
+                    view_id.dump() for view_id in self.topological_sort_implements(resource.implements)
+                ]
             except ToolkitCycleError as e:
                 warning = MediumSeverityWarning(f"Failed to sort implements for view {resource.as_id()}: {e}")
                 warning.print_warning(console=self.console)
@@ -644,78 +664,8 @@ class ViewLoader(ResourceLoader[ViewId, ViewApply, View, ViewApplyList, ViewList
             if self._is_auto_retryable(e1):
                 # Fallback to creating one by one if the error is auto-retryable.
                 return self._fallback_create_one_by_one(items, e1)
-            elif self._is_false_not_exists(e1, {item.as_id() for item in items}):
-                return self._try_to_recover_coupled(items, e1)
             raise
 
-    @staticmethod
-    def _is_false_not_exists(e: CogniteAPIError, request_views: set[ViewId]) -> bool:
-        if "not exist" not in e.message and 400 <= e.code < 500:
-            return False
-        results = re.findall(r"'([a-zA-Z0-9_-]+):([a-zA-Z0-9_]+)/([.a-zA-Z0-9_-]+)'", e.message)
-        if not results:
-            # No view references in the message
-            return False
-        error_message_views = {ViewId(*result) for result in results}
-        return error_message_views.issubset(request_views)
-
-    def _try_to_recover_coupled(self, items: Sequence[ViewApply], original_error: CogniteAPIError) -> ViewList:
-        """The /models/views endpoint can give faulty 400 about missing views that are part of the request.
-
-        This method tries to recover from such errors by identifying the strongly connected components in the graph
-        defined by the implements and through properties of the views. We then create the components in topological
-        order.
-
-        Args:
-            items: The items that failed to create.
-            original_error: The original error that was raised. If the problem is not recoverable, this error is raised.
-
-        Returns:
-            The views that were created.
-
-        """
-        views_by_id = {self.get_id(item): item for item in items}
-        parents_by_child: dict[ViewId, set[ViewId]] = {
-            view_id: {parent for parent in view.implements or [] if parent in views_by_id}
-            for view_id, view in views_by_id.items()
-        }
-        # Check for cycles in the implements graph
-        try:
-            TopologicalSorter(parents_by_child).static_order()
-        except CycleError as e:
-            raise ToolkitCycleError(f"Failed to deploy views. This likely due to a cycle in implements. {e.args[1]}")
-
-        dependencies_by_id: dict[ViewId, set[ViewId]] = defaultdict(set)
-        for view_id, view in views_by_id.items():
-            dependencies_by_id[view_id].update([parent for parent in view.implements or [] if parent in views_by_id])
-            for properties in (view.properties or {}).values():
-                if isinstance(properties, ReverseDirectRelationApply):
-                    if isinstance(properties.through.source, ViewId) and properties.through.source in views_by_id:
-                        dependencies_by_id[view_id].add(properties.through.source)
-
-        LowSeverityWarning(
-            f"Failed to create {len(items)} views: {escape(original_error.message)}.\nAttempting to recover..."
-        ).print_warning(include_timestamp=True, console=self.console)
-        created = ViewList([])
-        for strongly_connected in tarjan(dependencies_by_id):
-            to_create = [views_by_id[view_id] for view_id in strongly_connected]
-            try:
-                created_set = self.client.data_modeling.views.apply(to_create)
-            except CogniteAPIError as error:
-                self.client.data_modeling.views.delete(created.as_ids())
-                HighSeverityWarning(
-                    f"Recovering attempt failed. Could not create views {self.get_ids(to_create)}: "
-                    f"{escape(error.message)}.\n Raising original error."
-                ).print_warning(console=self.console)
-                raise original_error
-            created.extend(created_set)
-        message = f"Recovery attempt succeeded. Created {len(created)} views."
-        if self.console:
-            self.console.print(message)
-        else:
-            print(message)
-        return created
-
     @staticmethod
     def _is_auto_retryable(e: CogniteAPIError) -> bool:
         return isinstance(e.extra, dict) and "isAutoRetryable" in e.extra and e.extra["isAutoRetryable"]
@@ -732,8 +682,8 @@ class ViewLoader(ResourceLoader[ViewId, ViewApply, View, ViewApplyList, ViewList
             try:
                 created = self.client.data_modeling.views.apply(item)
             except CogniteAPIError as e2:
-                e2.failed.extend(items[no + 1 :])
-                e2.successful.extend(created_list)
+                e2.failed = tuple(e2.failed) + tuple(items[no + 1 :])
+                e2.successful = tuple(e2.successful) + tuple(created_list)
                 raise e2 from e1
             else:
                 created_list.append(created)
@@ -775,95 +725,9 @@ class ViewLoader(ResourceLoader[ViewId, ViewApply, View, ViewApplyList, ViewList
     ) -> Iterable[View]:
         return iter(self.client.data_modeling.views(space=space))
 
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        spec = super().get_write_cls_parameter_spec()
-        # The Filter class in the SDK class View implementation is deviating from the API.
-        # So we need to modify the spec to match the API.
-        parameter_path = ("filter",)
-        length = len(parameter_path)
-        for item in spec:
-            if len(item.path) >= length + 1 and item.path[:length] == parameter_path[:length]:
-                # Add extra ANY_STR layer
-                # The spec class is immutable, so we use this trick to modify it.
-                is_has_data_filter = item.path[1] in ["containers", "views"]
-                if is_has_data_filter:
-                    # Special handling of the HasData filter that deviates in SDK implementation from API Spec.
-                    object.__setattr__(item, "path", (*item.path[:length], ANY_STR, item.path[length + 1 :]))
-                else:
-                    object.__setattr__(item, "path", (*item.path[:length], ANY_STR, *item.path[length:]))
-
-        spec.add(ParameterSpec(("filter", ANY_STR), frozenset({"dict"}), is_required=False, _is_nullable=False))
-        # The following types are used by the SDK to load the correct class. They are not part of the init,
-        # so we need to add it manually.
-        spec.update(
-            ParameterSpecSet(
-                {
-                    ParameterSpec(
-                        ("implements", ANY_INT, "type"), frozenset({"str"}), is_required=True, _is_nullable=False
-                    ),
-                    ParameterSpec(
-                        ("properties", ANY_STR, "connectionType"),
-                        frozenset({"str"}),
-                        is_required=True,
-                        _is_nullable=False,
-                    ),
-                    ParameterSpec(
-                        ("properties", ANY_STR, "source", "type"),
-                        frozenset({"str"}),
-                        is_required=True,
-                        _is_nullable=False,
-                    ),
-                    ParameterSpec(
-                        ("properties", ANY_STR, "container", "type"),
-                        frozenset({"str"}),
-                        is_required=True,
-                        _is_nullable=False,
-                    ),
-                    ParameterSpec(
-                        ("properties", ANY_STR, "edgeSource", "type"),
-                        frozenset({"str"}),
-                        is_required=True,
-                        _is_nullable=False,
-                    ),
-                    ParameterSpec(
-                        ("properties", ANY_STR, "through", "source", "type"),
-                        frozenset({"str"}),
-                        is_required=True,
-                        _is_nullable=False,
-                    ),
-                    ParameterSpec(
-                        # In the SDK, this is called "property"
-                        ("properties", ANY_STR, "through", "identifier"),
-                        frozenset({"str"}),
-                        is_required=True,
-                        _is_nullable=False,
-                    ),
-                    # Filters are complex, so we do not attempt to give any more specific spec.
-                    ParameterSpec(
-                        ("filter", ANYTHING),
-                        frozenset({"str"}),
-                        is_required=True,
-                        _is_nullable=False,
-                    ),
-                }
-            )
-        )
-        spec.discard(
-            ParameterSpec(
-                # The API spec calls this "identifier", while the SDK calls it "property".
-                ("properties", ANY_STR, "through", "property"),
-                frozenset({"str"}),
-                is_required=True,
-                _is_nullable=False,
-            )
-        )
-        return spec
-
     @classmethod
     def as_str(cls, id: ViewId) -> str:
-        return to_directory_compatible(id.external_id)
+        return sanitize_filename(id.external_id)
 
     def _lookup_views(self, view_ids: list[ViewId]) -> dict[ViewId, View]:
         """Looks up views by their IDs and caches them."""
@@ -874,14 +738,51 @@ class ViewLoader(ResourceLoader[ViewId, ViewApply, View, ViewApplyList, ViewList
                 self._view_by_id[view.as_id()] = view
         return {view_id: self._view_by_id[view_id] for view_id in view_ids if view_id in self._view_by_id}
 
-    def topological_sort(self, view_ids: list[ViewId]) -> list[ViewId]:
+    def get_readonly_properties(self, view_id: ViewId) -> set[str]:
+        """Retrieve the set of read-only properties for a given view."""
+
+        readonly_properties: set[str] = set()
+
+        # Retrieve the view to check its properties
+        view = self._lookup_views([view_id]).get(view_id)
+        if view is None:
+            return readonly_properties
+
+        # Check each property in the view
+        for property_identifier, property in view.properties.items():
+            if isinstance(
+                property, MappedProperty
+            ) and property.container_property_identifier in constants.READONLY_CONTAINER_PROPERTIES.get(
+                property.container, set()
+            ):
+                readonly_properties.add(property_identifier)
+        return readonly_properties
+
+    def _build_view_implements_dependencies(
+        self, view_by_ids: dict[ViewId, View], include: set[ViewId] | None = None
+    ) -> dict[ViewId, set[ViewId]]:
+        """Build a dependency graph based on view implements relationships.
+
+        Args:
+            view_by_ids: Mapping of view IDs to View objects
+            include: Optional set of view IDs to include in the dependencies, if None, all views are included.
+
+        Returns:
+            Dictionary mapping each view ID to the set of view IDs it depends on (implements)
+        """
+        dependencies: dict[ViewId, set[ViewId]] = {}
+        for view_id, view in view_by_ids.items():
+            dependencies[view_id] = set()
+            for implemented_view_id in view.implements or []:
+                if include is None or implemented_view_id in include:
+                    dependencies[view_id].add(implemented_view_id)
+        return dependencies
+
+    def topological_sort_implements(self, view_ids: list[ViewId]) -> list[ViewId]:
         """Sorts the views in topological order based on their implements and through properties."""
         view_by_ids = self._lookup_views(view_ids)
-        parents_by_child: dict[ViewId, set[ViewId]] = {}
-        for child, view in view_by_ids.items():
-            parents_by_child[child] = set()
-            for parent in view.implements or []:
-                parents_by_child[child].add(parent)
+        parents_by_child = self._build_view_implements_dependencies(view_by_ids)
+
         try:
             sorted_views = list(TopologicalSorter(parents_by_child).static_order())
         except CycleError as e:
@@ -891,17 +792,63 @@ class ViewLoader(ResourceLoader[ViewId, ViewApply, View, ViewApplyList, ViewList
 
         return sorted_views
 
+    def topological_sort_container_constraints(self, view_ids: list[ViewId]) -> list[ViewId]:
+        """Sorts the views in topological order based on their container constraints."""
+
+        view_by_ids = self._lookup_views(view_ids)
+        if missing_view_ids := set(view_ids) - set(view_by_ids.keys()):
+            MediumSeverityWarning(
+                f"Views {missing_view_ids} not found or you don't have permission to access them, skipping dependency check."
+            ).print_warning(console=self.console)
+            return view_ids
+
+        view_to_containers: dict[ViewId, set[ContainerId]] = {}
+        container_to_views: defaultdict[ContainerId, set[ViewId]] = defaultdict(set)
+        for view_id, view in view_by_ids.items():
+            view_to_containers[view_id] = view.referenced_containers()
+            for container_id in view_to_containers[view_id]:
+                container_to_views[container_id].add(view_id)
+
+        container_crud = ContainerCRUD.create_loader(self.client)
+        container_dependencies_by_id = container_crud._find_direct_and_indirect_container_dependencies(
+            list(container_to_views.keys())
+        )
+
+        # First, add view dependencies based on implements relationships
+        view_dependencies = self._build_view_implements_dependencies(view_by_ids, set(view_to_containers.keys()))
+
+        # Then, add view dependencies based on mapped container constraints
+        for view_id, mapped_containers in view_to_containers.items():
+            for container_id in mapped_containers:
+                # Get all containers this container depends on
+                if container_id not in container_dependencies_by_id:
+                    continue
+                for required_container in container_dependencies_by_id[container_id]:
+                    if required_container not in container_to_views:
+                        continue
+                    # If this view already implements the required container, the requirement is self-satisfied
+                    # and we don't need to depend on other views that also implement it (they are peers).
+                    if required_container in mapped_containers:
+                        continue
+                    # This view doesn't implement the required container, so depend on all views that do
+                    view_dependencies[view_id].update(container_to_views[required_container])
+        try:
+            sorted_views = list(TopologicalSorter(view_dependencies).static_order())
+        except CycleError as e:
+            raise ToolkitCycleError(
+                f"Failed to sort views topologically. This likely due to a cycle in implements. {e.args[1]}"
+            )
+        return sorted_views
+
 
 @final
-class DataModelLoader(ResourceLoader[DataModelId, DataModelApply, DataModel, DataModelApplyList, DataModelList]):
-    folder_name = "data_models"
-    filename_pattern = r"^.*datamodel$"
+class DataModelCRUD(ResourceCRUD[DataModelId, DataModelApply, DataModel]):
+    folder_name = "data_modeling"
     resource_cls = DataModel
     resource_write_cls = DataModelApply
-    list_cls = DataModelList
-    list_write_cls = DataModelApplyList
     kind = "DataModel"
-    dependencies = frozenset({SpaceLoader, ViewLoader})
+    dependencies = frozenset({SpaceCRUD, ViewCRUD})
+    yaml_cls = DataModelYAML
     _doc_url = "Data-models/operation/createDataModels"
 
     @property
@@ -923,7 +870,7 @@ class DataModelLoader(ResourceLoader[DataModelId, DataModelApply, DataModel, Dat
             else DataModelsAcl.Scope.All()
         )
 
-        return DataModelsAcl(actions, scope)  # type: ignore[arg-type]
+        return DataModelsAcl(actions, scope)
 
     @classmethod
     def get_id(cls, item: DataModelApply | DataModel | dict) -> DataModelId:
@@ -939,13 +886,13 @@ class DataModelLoader(ResourceLoader[DataModelId, DataModelApply, DataModel, Dat
         return id.dump(include_type=False)
 
     @classmethod
-    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader], Hashable]]:
+    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
         if "space" in item:
-            yield SpaceLoader, item["space"]
+            yield SpaceCRUD, item["space"]
         for view in item.get("views", []):
             if in_dict(("space", "externalId"), view):
                 yield (
-                    ViewLoader,
+                    ViewCRUD,
                     ViewId(view["space"], view["externalId"], str(v) if (v := view.get("version")) else None),
                 )
 
@@ -975,13 +922,13 @@ class DataModelLoader(ResourceLoader[DataModelId, DataModelApply, DataModel, Dat
             return diff_list_identifiable(local, cdf, get_identifier=dm_identifier)
         return super().diff_list(local, cdf, json_path)
 
-    def create(self, items: DataModelApplyList) -> DataModelList:
+    def create(self, items: Sequence[DataModelApply]) -> DataModelList:
         return self.client.data_modeling.data_models.apply(items)
 
     def retrieve(self, ids: SequenceNotStr[DataModelId]) -> DataModelList:
         return self.client.data_modeling.data_models.retrieve(cast(Sequence, ids))
 
-    def update(self, items: DataModelApplyList) -> DataModelList:
+    def update(self, items: Sequence[DataModelApply]) -> DataModelList:
         updated = self.create(items)
         # There is a bug in the API not raising an exception if view is removed from a data model.
         # So we check here that the update was fixed.
@@ -1019,32 +966,22 @@ class DataModelLoader(ResourceLoader[DataModelId, DataModelApply, DataModel, Dat
     ) -> Iterable[DataModel]:
         return iter(self.client.data_modeling.data_models(space=space, include_global=False))
 
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        spec = super().get_write_cls_parameter_spec()
-        # ViewIds have the type set in the API Spec, but this is hidden in the SDK classes,
-        # so we need to add it manually.
-        spec.add(ParameterSpec(("views", ANY_INT, "type"), frozenset({"str"}), is_required=True, _is_nullable=False))
-        return spec
-
     @classmethod
     def as_str(cls, id: DataModelId) -> str:
-        return to_directory_compatible(id.external_id)
+        return sanitize_filename(id.external_id)
 
 
 @final
-class NodeLoader(ResourceContainerLoader[NodeId, NodeApply, Node, NodeApplyList, NodeList]):
+class NodeCRUD(ResourceContainerCRUD[NodeId, NodeApply, Node]):
     item_name = "nodes"
-    folder_name = "data_models"
-    filename_pattern = r"^.*node$"
+    folder_name = "data_modeling"
     resource_cls = Node
     resource_write_cls = NodeApply
-    list_cls = NodeList
-    list_write_cls = NodeApplyList
     kind = "Node"
-    dependencies = frozenset({SpaceLoader, ViewLoader, ContainerLoader})
+    yaml_cls = NodeYAML
+    dependencies = frozenset({SpaceCRUD, ViewCRUD, ContainerCRUD})
     _doc_url = "Instances/operation/applyNodeAndEdges"
+    sub_folder_name = "nodes"
 
     def __init__(
         self,
@@ -1095,14 +1032,14 @@ class NodeLoader(ResourceContainerLoader[NodeId, NodeApply, Node, NodeApplyList,
         return id.dump()
 
     @classmethod
-    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader], Hashable]]:
+    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
         if "space" in item:
-            yield SpaceLoader, item["space"]
+            yield SpaceCRUD, item["space"]
         for source in item.get("sources", []):
             if (identifier := source.get("source")) and isinstance(identifier, dict):
                 if identifier.get("type") == "view" and in_dict(("space", "externalId", "version"), identifier):
                     yield (
-                        ViewLoader,
+                        ViewCRUD,
                         ViewId(
                             identifier["space"],
                             identifier["externalId"],
@@ -1110,7 +1047,7 @@ class NodeLoader(ResourceContainerLoader[NodeId, NodeApply, Node, NodeApplyList,
                         ),
                     )
                 elif identifier.get("type") == "container" and in_dict(("space", "externalId"), identifier):
-                    yield ContainerLoader, ContainerId(identifier["space"], identifier["externalId"])
+                    yield ContainerCRUD, ContainerId(identifier["space"], identifier["externalId"])
 
     def dump_resource(self, resource: Node, local: dict[str, Any] | None = None) -> dict[str, Any]:
         # CDF resource does not have properties set, so we need to do a lookup
@@ -1140,7 +1077,7 @@ class NodeLoader(ResourceContainerLoader[NodeId, NodeApply, Node, NodeApplyList,
 
         return dumped
 
-    def create(self, items: NodeApplyList) -> NodeApplyResultList:
+    def create(self, items: Sequence[NodeApply]) -> NodeApplyResultList:
         result = self.client.data_modeling.instances.apply(
             # Note replace should never be relevant as Toolkit always checks whether the node exists before applying.
             nodes=items,
@@ -1152,7 +1089,7 @@ class NodeLoader(ResourceContainerLoader[NodeId, NodeApply, Node, NodeApplyList,
     def retrieve(self, ids: SequenceNotStr[NodeId]) -> NodeList:
         return self.client.data_modeling.instances.retrieve(nodes=cast(Sequence, ids), sources=self.view_id).nodes
 
-    def update(self, items: NodeApplyList) -> NodeApplyResultList:
+    def update(self, items: Sequence[NodeApply]) -> NodeApplyResultList:
         result = self.client.data_modeling.instances.apply(
             nodes=items, auto_create_direct_relations=True, replace=False
         )
@@ -1184,41 +1121,19 @@ class NodeLoader(ResourceContainerLoader[NodeId, NodeApply, Node, NodeApplyList,
         # Nodes will be deleted in .delete call.
         return 0
 
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        node_spec = super().get_write_cls_parameter_spec()
-        # This is a deviation between the SDK and the API
-        node_spec.add(ParameterSpec(("instanceType",), frozenset({"str"}), is_required=False, _is_nullable=False))
-        node_spec.add(
-            ParameterSpec(
-                ("sources", ANY_INT, "source", "type"),
-                frozenset({"str"}),
-                is_required=True,
-                _is_nullable=False,
-            )
-        )
-        return ParameterSpecSet(node_spec, spec_name=cls.__name__)
-
     @classmethod
     def as_str(cls, id: NodeId) -> str:
-        return to_directory_compatible(f"{id.space}_{id.external_id}")
+        return sanitize_filename(f"{id.space}_{id.external_id}")
 
 
-class GraphQLLoader(
-    ResourceContainerLoader[
-        DataModelId, GraphQLDataModelWrite, GraphQLDataModel, GraphQLDataModelWriteList, GraphQLDataModelList
-    ]
-):
-    folder_name = "data_models"
-    filename_pattern = r"^.*GraphQLSchema"
+class GraphQLCRUD(ResourceContainerCRUD[DataModelId, GraphQLDataModelWrite, GraphQLDataModel]):
+    folder_name = "data_modeling"
     resource_cls = GraphQLDataModel
     resource_write_cls = GraphQLDataModelWrite
-    list_cls = GraphQLDataModelList
-    list_write_cls = GraphQLDataModelWriteList
     kind = "GraphQLSchema"
-    dependencies = frozenset({SpaceLoader, ContainerLoader})
+    dependencies = frozenset({SpaceCRUD, ContainerCRUD})
     item_name = "views"
+    yaml_cls = GraphQLDataModelYAML
     _doc_url = "Data-models/operation/createDataModels"
     _hash_name = "CDFToolkitHash:"
 
@@ -1260,9 +1175,9 @@ class GraphQLLoader(
         )
 
     @classmethod
-    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader], Hashable]]:
+    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
         if "space" in item:
-            yield SpaceLoader, item["space"]
+            yield SpaceCRUD, item["space"]
 
     def safe_read(self, filepath: Path | str) -> str:
         # The version is a string, but the user often writes it as an int.
@@ -1328,7 +1243,7 @@ class GraphQLLoader(
             dumped["graphqlFile"] = match.group(3)
         return dumped
 
-    def create(self, items: GraphQLDataModelWriteList) -> list[DMLApplyResult]:
+    def create(self, items: Sequence[GraphQLDataModelWrite]) -> list[DMLApplyResult]:
         creation_order = self._topological_sort(items)
 
         created_list: list[DMLApplyResult] = []
@@ -1359,7 +1274,7 @@ class GraphQLLoader(
         result = self.client.data_modeling.data_models.retrieve(list(ids), inline_views=False)
         return GraphQLDataModelList([GraphQLDataModel._load(d.dump()) for d in result])
 
-    def update(self, items: GraphQLDataModelWriteList) -> list[DMLApplyResult]:
+    def update(self, items: Sequence[GraphQLDataModelWrite]) -> list[DMLApplyResult]:
         return self.create(items)
 
     def delete(self, ids: SequenceNotStr[DataModelId]) -> int:
@@ -1384,7 +1299,7 @@ class GraphQLLoader(
     def drop_data(self, ids: SequenceNotStr[DataModelId]) -> int:
         return self.delete(ids)
 
-    def _topological_sort(self, items: GraphQLDataModelWriteList) -> list[GraphQLDataModelWrite]:
+    def _topological_sort(self, items: Sequence[GraphQLDataModelWrite]) -> list[GraphQLDataModelWrite]:
         to_sort = {item.as_id(): item for item in items}
         dependencies: dict[DataModelId, set[DataModelId]] = {}
         for item in items:
@@ -1407,16 +1322,14 @@ class GraphQLLoader(
 
 
 @final
-class EdgeLoader(ResourceContainerLoader[EdgeId, EdgeApply, Edge, EdgeApplyList, EdgeList]):
+class EdgeCRUD(ResourceContainerCRUD[EdgeId, EdgeApply, Edge]):
     item_name = "edges"
-    folder_name = "data_models"
-    filename_pattern = r"^.*edge"
+    folder_name = "data_modeling"
     resource_cls = Edge
     resource_write_cls = EdgeApply
-    list_cls = EdgeList
-    list_write_cls = EdgeApplyList
     kind = "Edge"
-    dependencies = frozenset({SpaceLoader, ViewLoader, ContainerLoader, NodeLoader})
+    yaml_cls = EdgeYAML
+    dependencies = frozenset({SpaceCRUD, ViewCRUD, ContainerCRUD, NodeCRUD})
     _doc_url = "Instances/operation/applyNodeAndEdges"
 
     @property
@@ -1457,14 +1370,14 @@ class EdgeLoader(ResourceContainerLoader[EdgeId, EdgeApply, Edge, EdgeApplyList,
         return id.dump()
 
     @classmethod
-    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader], Hashable]]:
+    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
         if "space" in item:
-            yield SpaceLoader, item["space"]
+            yield SpaceCRUD, item["space"]
         for source in item.get("sources", []):
             if (identifier := source.get("source")) and isinstance(identifier, dict):
                 if identifier.get("type") == "view" and in_dict(("space", "externalId", "version"), identifier):
                     yield (
-                        ViewLoader,
+                        ViewCRUD,
                         ViewId(
                             identifier["space"],
                             identifier["externalId"],
@@ -1472,12 +1385,12 @@ class EdgeLoader(ResourceContainerLoader[EdgeId, EdgeApply, Edge, EdgeApplyList,
                         ),
                     )
                 elif identifier.get("type") == "container" and in_dict(("space", "externalId"), identifier):
-                    yield ContainerLoader, ContainerId(identifier["space"], identifier["externalId"])
+                    yield ContainerCRUD, ContainerId(identifier["space"], identifier["externalId"])
 
         for key in ["startNode", "endNode", "type"]:
             if node_ref := item.get(key):
                 if isinstance(node_ref, dict) and in_dict(("space", "externalId"), node_ref):
-                    yield NodeLoader, NodeId(node_ref["space"], node_ref["externalId"])
+                    yield NodeCRUD, NodeId(node_ref["space"], node_ref["externalId"])
 
     def dump_resource(self, resource: Edge, local: dict[str, Any] | None = None) -> dict[str, Any]:
         # CDF resource does not have properties set, so we need to do a lookup
@@ -1504,7 +1417,7 @@ class EdgeLoader(ResourceContainerLoader[EdgeId, EdgeApply, Edge, EdgeApplyList,
 
         return dumped
 
-    def create(self, items: EdgeApplyList) -> EdgeApplyResultList:
+    def create(self, items: Sequence[EdgeApply]) -> EdgeApplyResultList:
         result = self.client.data_modeling.instances.apply(
             edges=items, auto_create_direct_relations=True, replace=False
         )
@@ -1513,7 +1426,7 @@ class EdgeLoader(ResourceContainerLoader[EdgeId, EdgeApply, Edge, EdgeApplyList,
     def retrieve(self, ids: SequenceNotStr[EdgeId]) -> EdgeList:
         return self.client.data_modeling.instances.retrieve(nodes=cast(Sequence, ids)).edges
 
-    def update(self, items: EdgeApplyList) -> EdgeApplyResultList:
+    def update(self, items: Sequence[EdgeApply]) -> EdgeApplyResultList:
         result = self.client.data_modeling.instances.apply(
             edges=items, auto_create_direct_relations=False, replace=True
         )
@@ -1543,22 +1456,6 @@ class EdgeLoader(ResourceContainerLoader[EdgeId, EdgeApply, Edge, EdgeApplyList,
         # Edges will be deleted in .delete call.
         return 0
 
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        node_spec = super().get_write_cls_parameter_spec()
-        # This is a deviation between the SDK and the API
-        node_spec.add(ParameterSpec(("instanceType",), frozenset({"str"}), is_required=False, _is_nullable=False))
-        node_spec.add(
-            ParameterSpec(
-                ("sources", ANY_INT, "source", "type"),
-                frozenset({"str"}),
-                is_required=True,
-                _is_nullable=False,
-            )
-        )
-        return ParameterSpecSet(node_spec, spec_name=cls.__name__)
-
     def diff_list(
         self, local: list[Any], cdf: list[Any], json_path: tuple[str | int, ...]
     ) -> tuple[dict[int, int], list[int]]: