cognite-neat 1.0.10__py3-none-any.whl → 1.0.11__py3-none-any.whl

cognite/neat/__init__.py

@@ -1,7 +1,7 @@
 from cognite.neat._v0.core._utils.auth import get_cognite_client
 
-from ._config import NeatConfig
+from ._config import NeatConfig, get_neat_config_from_file
 from ._session import NeatSession
 from ._version import __version__
 
-__all__ = ["NeatConfig", "NeatSession", "__version__", "get_cognite_client"]
+__all__ = ["NeatConfig", "NeatSession", "__version__", "get_cognite_client", "get_neat_config_from_file"]

cognite/neat/_config.py

@@ -121,7 +121,7 @@ class AlphaFlagConfig(ConfigModel):
 
 
 class NeatConfig(ConfigModel):
-    """Configuration for a custom profile."""
+    """NeatSession configuration."""
 
     profile: str
     validation: ValidationConfig
@@ -139,7 +139,19 @@ class NeatConfig(ConfigModel):
 
     @classmethod
     def create_predefined(cls, profile: PredefinedProfile = "legacy-additive") -> "NeatConfig":
-        """Create NeatConfig from internal profiles."""
+        """Create NeatConfig from internal profiles.
+
+        Args:
+            profile: Profile name to use. Defaults to "legacy-additive".
+
+        !!! note "Predefined Profiles"
+            The following predefined profiles are available:
+
+            - `legacy-additive`: Additive modeling with legacy validation rules.
+            - `legacy-rebuild`: Rebuild modeling with legacy validation rules.
+            - `deep-additive`: Additive modeling with deep validation rules.
+            - `deep-rebuild`: Rebuild modeling with deep validation rules.
+        """
         available_profiles = internal_profiles()
         if profile not in available_profiles:
             raise UserInputError(

cognite/neat/_data_model/_analysis.py

@@ -1,15 +1,18 @@
-from graphlib import TopologicalSorter
+from itertools import chain
+from typing import Literal, TypeAlias
 
-from cognite.neat._data_model.models.dms._constraints import RequiresConstraintDefinition
-from cognite.neat._data_model.models.dms._container import ContainerPropertyDefinition, ContainerRequest
+from pyparsing import cached_property
+
+from cognite.neat._data_model._snapshot import SchemaSnapshot
+from cognite.neat._data_model.models.dms._container import ContainerRequest
 from cognite.neat._data_model.models.dms._data_types import DirectNodeRelation
+from cognite.neat._data_model.models.dms._limits import SchemaLimits
 from cognite.neat._data_model.models.dms._references import (
     ContainerDirectReference,
     ContainerReference,
     ViewDirectReference,
     ViewReference,
 )
-from cognite.neat._data_model.models.dms._schema import RequestSchema
 from cognite.neat._data_model.models.dms._view_property import (
     EdgeProperty,
     ReverseDirectRelationProperty,
@@ -17,180 +20,326 @@ from cognite.neat._data_model.models.dms._view_property import (
     ViewRequestProperty,
 )
 from cognite.neat._data_model.models.dms._views import ViewRequest
+from cognite.neat._utils.useful_types import ModusOperandi
 
+# Type aliases for better readability
+ViewsByReference: TypeAlias = dict[ViewReference, ViewRequest]
+ContainersByReference: TypeAlias = dict[ContainerReference, ContainerRequest]
+AncestorsByReference: TypeAlias = dict[ViewReference, set[ViewReference]]
+ReverseToDirectMapping: TypeAlias = dict[
+    tuple[ViewReference, str], tuple[ViewReference, ContainerDirectReference | ViewDirectReference]
+]
+ConnectionEndNodeTypes: TypeAlias = dict[tuple[ViewReference, str], ViewReference | None]
 
-class DataModelAnalysis:
-    def __init__(
-        self,
-        physical: RequestSchema | None = None,
-    ) -> None:
-        self._physical = physical
 
-    @property
-    def physical(self) -> RequestSchema:
-        if self._physical is None:
-            raise ValueError("Physical Data Model is required for this analysis")
-        return self._physical
+ResourceSource = Literal["auto", "merged", "cdf", "both"]
 
-    def referenced_views(self, include_connection_end_node_types: bool = False) -> set[ViewReference]:
-        """Get all referenced views in the physical data model."""
-        referenced_views = set()
 
-        for view in self.physical.views:
-            referenced_views.add(view.as_reference())
-            if view.implements:
-                for implement in view.implements:
-                    referenced_views.add(implement)
+class ValidationResources:
+    def __init__(
+        self, modus_operandi: ModusOperandi, local: SchemaSnapshot, cdf: SchemaSnapshot, limits: SchemaLimits
+    ) -> None:
+        self._modus_operandi = modus_operandi
+        self.limits = limits
 
-        if include_connection_end_node_types:
-            referenced_views |= {view for view in self.connection_end_node_types.values() if view is not None}
+        self.local = local
+        self.cdf = cdf
+
+        if self._modus_operandi == "additive":
+            self.merged = self.local.merge(self.cdf)
+        elif self._modus_operandi == "rebuild":
+            self.merged = local.model_copy(deep=True)
+        else:
+            raise RuntimeError(f"ValidationResources: Unknown modus_operandi: {self._modus_operandi}. This is a bug!")
+
+        # need this shortcut for easier access and also to avoid mypy to complains
+        self.merged_data_model = self.merged.data_model[next(iter(self.merged.data_model.keys()))]
+
+    def select_view(
+        self, view_ref: ViewReference, property_: str | None = None, source: ResourceSource = "auto"
+    ) -> ViewRequest | None:
+        """Select view definition based on source strategy and optionally filter by property.
+
+        Selection prioritize merged view over CDF if both are available, as merged view represents the effective
+        definition which will be a result when the local schema is deployed to CDF.
+
+
+        Args:
+            view_ref: The view to select
+            property_: Optional property name to filter views that contain this property
+            source: The source strategy to use, options are: "auto", "local", "cdf", "both", where "auto" means
+                that the selection is driven by the modus_operandi of the validation resources.
+
+        Returns:
+            The selected ViewRequest or None if not found.
+        """
+
+        check_merged, check_cdf = self._resolve_resource_sources(view_ref, source)
+
+        merged_view = self.merged.views.get(view_ref) if check_merged else None
+        cdf_view = self.cdf.views.get(view_ref) if check_cdf else None
+
+        if property_ is None:
+            return merged_view or cdf_view
+
+        # Filtering based on the property presence
+        # Try views with the property first, then any available view where merged view is prioritized
+        candidates = chain(
+            (v for v in (merged_view, cdf_view) if v and v.properties and property_ in v.properties),
+            (v for v in (merged_view, cdf_view) if v),
+        )
+
+        return next(candidates, None)
+
+    def select_container(
+        self, container_ref: ContainerReference, property_: str | None = None, source: ResourceSource = "auto"
+    ) -> ContainerRequest | None:
+        """Select container definition based on source strategy and optionally filter by property.
+
+        Selection prioritize merged container over CDF if both are available, as merged container represents
+        the effective definition which will be a result when the local schema is deployed to CDF.
+
+        Args:
+            container_ref: The container to select
+            property_: Optional property name to filter containers that contain this property
+            source: The source strategy to use, options are: "auto", "local", "cdf", "both", where "auto" means
+                that the selection is driven by the modus_operandi of the validation resources.
+
+        Returns:
+            The selected ContainerRequest or None if not found.
+        """
+
+        check_merged, check_cdf = self._resolve_resource_sources(container_ref, source)
+
+        merged_container = self.merged.containers.get(container_ref) if check_merged else None
+        cdf_container = self.cdf.containers.get(container_ref) if check_cdf else None
+
+        if property_ is None:
+            return merged_container or cdf_container
+
+        # Try containers with the property first, then any available container
+        candidates = chain(
+            (c for c in (merged_container, cdf_container) if c and c.properties and property_ in c.properties),
+            (c for c in (merged_container, cdf_container) if c),
+        )
+
+        return next(candidates, None)
+
+    def _resolve_resource_sources(
+        self, resource_ref: ViewReference | ContainerReference, source: ResourceSource
+    ) -> tuple[bool, bool]:
+        """
+        Determine which resource sources (merged and/or CDF) to check based on the source parameter.
+
+        Args:
+            resource_ref: The resource reference to check (ViewReference or ContainerReference)
+            source: The source strategy to use
+
+        Returns:
+            Tuple of (check_merged, check_cdf) booleans indicating which sources to check
+        """
+        if source == "auto":
+            # Auto mode: driven by modus_operandi
+            # In "additive" mode or for resources outside local space, check both merged and CDF
+            # In "rebuild" mode for resources in local space, check only merged == local
+            check_merged = True
+            check_cdf = resource_ref.space != self.merged_data_model.space or self._modus_operandi == "additive"
+        elif source == "merged":
+            check_merged = True
+            check_cdf = False
+        elif source == "cdf":
+            check_merged = False
+            check_cdf = True
+        elif source == "both":
+            check_merged = True
+            check_cdf = True
+        else:
+            raise RuntimeError(f"_resolve_resource_sources: Unknown source: {source}. This is a bug!")
+
+        return check_merged, check_cdf
+
+    @cached_property
+    def ancestors_by_view(self) -> dict[ViewReference, list[ViewReference]]:
+        """
+        Create a mapping of each view to its list of ancestors.
+
+        Returns:
+            Dictionary mapping each ViewReference to its list of ancestor ViewReferences
+        """
+        ancestors_mapping: dict[ViewReference, list[ViewReference]] = {}
+
+        if not self.merged_data_model.views:
+            return ancestors_mapping
+
+        for view in self.merged_data_model.views:
+            ancestors_mapping[view] = self.view_ancestors(view)
+        return ancestors_mapping
+
+    def view_ancestors(
+        self, offspring: ViewReference, ancestors: list[ViewReference] | None = None, source: ResourceSource = "auto"
+    ) -> list[ViewReference]:
+        """
+        Recursively find all ancestors of a given view by traversing the implements hierarchy.
+        Handles branching to explore all possible ancestor paths.
+
+        Args:
+            offspring: The view to find ancestors for
+            ancestors: Accumulated list of ancestors (used internally for recursion)
+
+        Returns:
+            List of all ancestor ViewReferences
+        """
+        if ancestors is None:
+            ancestors = []
+
+        # Determine which view definition to use based on space and modus operandi
+
+        view_definition = self.select_view(view_ref=offspring, source=source)
+
+        # Base case: no view definition or no implements
+        if not view_definition or not view_definition.implements:
+            return ancestors
+
+        # Explore all parent branches
+        for parent in view_definition.implements:
+            if parent not in ancestors:
+                ancestors.append(parent)
+                # Recursively explore this branch
+                self.view_ancestors(parent, ancestors)
+
+        return ancestors
+
+    def is_ancestor(self, offspring: ViewReference, ancestor: ViewReference) -> bool:
+        return ancestor in self.ancestors_by_view.get(offspring, set())
+
+    @cached_property
+    def properties_by_view(self) -> dict[ViewReference, dict[str, ViewRequestProperty]]:
+        """Get a mapping of view references to their corresponding properties, both directly defined and inherited
+        from ancestor views through implements."""
 
-        return referenced_views
+        properties_mapping: dict[ViewReference, dict[str, ViewRequestProperty]] = {}
 
-    @property
-    def referenced_containers(self) -> set[ContainerReference]:
-        """Get all referenced containers in the physical data model."""
-        referenced_containers = set()
+        if self.merged_data_model.views:
+            for view_ref in self.merged_data_model.views:
+                view = self.select_view(view_ref)
+                # This should never happen, if it happens, it's a bug
+                if not view:
+                    raise RuntimeError(f"properties_by_view: View {view_ref!s} not found. This is a bug!")
 
-        for view in self.physical.views:
-            for property_ in view.properties.values():
-                if isinstance(property_, ViewCorePropertyRequest):
-                    referenced_containers.add(property_.container)
+                combined_properties: dict[str, ViewRequestProperty] = {}
+                ancestors = self.ancestors_by_view.get(view_ref, [])
+                # Start with properties from ancestor views
+                for ancestor_ref in reversed(ancestors):
+                    ancestor_view = self.select_view(ancestor_ref)
+                    if ancestor_view:
+                        combined_properties.update(ancestor_view.properties)
 
-        for container in self.physical.containers:
-            referenced_containers.add(container.as_reference())
-            if container.constraints:
-                for constraint in container.constraints.values():
-                    if isinstance(constraint, RequiresConstraintDefinition):
-                        referenced_containers.add(constraint.require)
+                # Finally, add properties from the current view, overriding any inherited ones
+                if view.properties:
+                    combined_properties.update(view.properties)
 
-        return referenced_containers
+                properties_mapping[view_ref] = combined_properties
 
-    def view_by_reference(self, include_inherited_properties: bool = True) -> dict[ViewReference, ViewRequest]:
-        """Get a mapping of view references to their corresponding ViewRequest objects."""
-        view_ancestors = self.ancestors_by_view(self.physical.views)
-
-        view_by_reference: dict[ViewReference, ViewRequest] = {
-            view.as_reference(): view.model_copy(deep=True) for view in self.physical.views
-        }
+        return properties_mapping
 
-        if include_inherited_properties:
-            for ref, view in view_by_reference.items():
-                for ancestor in view_ancestors.get(ref, set()):
-                    if ancestor_view := view_by_reference.get(ancestor):
-                        if ancestor_view.properties:
-                            view.properties.update(ancestor_view.properties)
-
-        return view_by_reference
+    @cached_property
+    def referenced_containers(self) -> set[ContainerReference]:
+        """Get a set of all container references used by the views in the local data model."""
 
-    @property
-    def properties_by_view(self) -> dict[ViewReference, dict[str, ViewRequestProperty]]:
-        """Get a mapping of view references to their corresponding properties."""
-        view_by_reference = self.view_by_reference(include_inherited_properties=False)
-        return {
-            view_ref: {prop_name: prop for prop_name, prop in view.properties.items()}
-            for view_ref, view in view_by_reference.items()
-        }
-
-    @staticmethod
-    def ancestors_by_view(views: list[ViewRequest]) -> dict[ViewReference, set[ViewReference]]:
-        """Get a mapping of each view to its ancestors in the physical data model."""
-        implements_by_view = DataModelAnalysis.implements_by_view(views)
-
-        # Topological sort to ensure that concepts include all ancestors
-        for view in list(TopologicalSorter(implements_by_view).static_order()):
-            if view not in implements_by_view:
-                continue
-            implements_by_view[view] |= {
-                grand_parent
-                for parent in implements_by_view[view]
-                for grand_parent in implements_by_view.get(parent, set())
-            }
+        referenced_containers: set[ContainerReference] = set()
 
-        return implements_by_view
+        if not self.merged_data_model.views:
+            return referenced_containers
 
-    @staticmethod
-    def implements_by_view(views: list[ViewRequest]) -> dict[ViewReference, set[ViewReference]]:
-        """Get a mapping of each view to the views it implements."""
-        implements_mapping: dict[ViewReference, set[ViewReference]] = {}
+        for view_ref in self.merged_data_model.views:
+            view = self.select_view(view_ref)
+            # This should never happen, if it happens, it's a bug
+            if not view:
+                raise RuntimeError(f"referenced_containers: View {view_ref!s} not found. This is a bug!")
 
-        for view in views:
-            view_ref = view.as_reference()
-            if view_ref not in implements_mapping:
-                implements_mapping[view_ref] = set()
-            if view.implements:
-                for implement in view.implements:
-                    implements_mapping[view_ref].add(implement)
+            if not view.properties:
+                continue
+            for property_ in view.properties.values():
+                if isinstance(property_, ViewCorePropertyRequest):
+                    referenced_containers.add(property_.container)
 
-        return implements_mapping
+        return referenced_containers
 
-    @property
-    def container_by_reference(self) -> dict[ContainerReference, ContainerRequest]:
-        """Get a mapping of container references to their corresponding ContainerRequest objects."""
-        return {container.as_reference(): container.model_copy(deep=True) for container in self.physical.containers}
+    @cached_property
+    def reverse_to_direct_mapping(
+        self,
+    ) -> dict[tuple[ViewReference, str], tuple[ViewReference, ContainerDirectReference | ViewDirectReference]]:
+        """Get a mapping of reverse direct relations to their corresponding source view and 'through' property."""
 
-    @property
-    def container_properties(self) -> dict[tuple[ContainerReference, str], ContainerPropertyDefinition]:
-        """Get a mapping from (container reference, property name) to the property definition."""
+        bidirectional_connections: dict[
+            tuple[ViewReference, str], tuple[ViewReference, ContainerDirectReference | ViewDirectReference]
+        ] = {}
+
+        if self.merged_data_model.views:
+            for view_ref in self.merged_data_model.views:
+                view = self.select_view(view_ref)
+
+                # This should never happen, if it happens, it's a bug
+                if not view:
+                    raise RuntimeError(f"reverse_to_direct_mapping: View {view_ref!s} not found. This is a bug!")
+
+                if not view.properties:
+                    continue
+                for prop_ref, property_ in view.properties.items():
+                    # reverse direct relation
+                    if isinstance(property_, ReverseDirectRelationProperty):
+                        bidirectional_connections[(view_ref, prop_ref)] = (
+                            property_.source,
+                            property_.through,
+                        )
 
-        return {
-            (container.as_reference(), prop_name): property_
-            for container in self.physical.containers
-            for prop_name, property_ in container.properties.items()
-        }
+        return bidirectional_connections
 
     @property
     def connection_end_node_types(self) -> dict[tuple[ViewReference, str], ViewReference | None]:
         """Get a mapping of view references to their corresponding ViewRequest objects."""
-        view_by_reference = self.view_by_reference(include_inherited_properties=False)
+
         connection_end_node_types: dict[tuple[ViewReference, str], ViewReference | None] = {}
-        container_properties = self.container_properties
 
-        for view_ref, view in view_by_reference.items():
-            if not view.properties:
-                continue
-            for prop_ref, property_ in view.properties.items():
-                # direct relation
-                if isinstance(property_, ViewCorePropertyRequest):
-                    # explicit set of end node type via 'source' which is View reference
-                    if property_.source:
+        if self.merged_data_model.views:
+            for view_ref in self.merged_data_model.views:
+                view = self.select_view(view_ref)
+                if not view:
+                    raise RuntimeError(f"View {view_ref!s} not found. This is a bug!")
+
+                if not view.properties:
+                    continue
+
+                for prop_ref, property_ in view.properties.items():
+                    # direct relation
+                    if isinstance(property_, ViewCorePropertyRequest):
+                        # explicit set of end node type via 'source' which is View reference
+                        if property_.source:
+                            connection_end_node_types[(view_ref, prop_ref)] = property_.source
+
+                        # implicit end node type via container property, without actual knowledge of end node type
+                        elif (
+                            (
+                                container := self.select_container(
+                                    property_.container, property_.container_property_identifier
+                                )
+                            )
+                            and (property_.container_property_identifier in container.properties)
+                            and (
+                                isinstance(
+                                    container.properties[property_.container_property_identifier].type,
+                                    DirectNodeRelation,
+                                )
+                            )
+                        ):
+                            connection_end_node_types[(view_ref, prop_ref)] = None
+
+                    # reverse direct relation
+                    elif isinstance(property_, ReverseDirectRelationProperty) and property_.source:
                         connection_end_node_types[(view_ref, prop_ref)] = property_.source
 
-                    # implicit end node type via container property, without actual knowledge of end node type
-                    elif (
-                        container_property := container_properties.get(
-                            (property_.container, property_.container_property_identifier)
-                        )
-                    ) and isinstance(container_property.type, DirectNodeRelation):
-                        connection_end_node_types[(view_ref, prop_ref)] = None
-
-                # reverse direct relation
-                if isinstance(property_, ReverseDirectRelationProperty) and property_.source:
-                    connection_end_node_types[(view_ref, prop_ref)] = property_.source
-
-                # edge property
-                if isinstance(property_, EdgeProperty) and property_.source:
-                    connection_end_node_types[(view_ref, prop_ref)] = property_.source
+                    # edge property
+                    elif isinstance(property_, EdgeProperty) and property_.source:
+                        connection_end_node_types[(view_ref, prop_ref)] = property_.source
 
         return connection_end_node_types
-
-    @property
-    def reverse_to_direct_mapping(
-        self,
-    ) -> dict[tuple[ViewReference, str], tuple[ViewReference, ContainerDirectReference | ViewDirectReference]]:
-        """Get a mapping of reverse direct relations to their corresponding source view and 'through' property."""
-        view_by_reference = self.view_by_reference(include_inherited_properties=False)
-        bidirectional_connections = {}
-
-        for view_ref, view in view_by_reference.items():
-            if not view.properties:
-                continue
-            for prop_ref, property_ in view.properties.items():
-                # reverse direct relation
-                if isinstance(property_, ReverseDirectRelationProperty):
-                    bidirectional_connections[(view_ref, prop_ref)] = (
-                        property_.source,
-                        property_.through,
-                    )
-
-        return bidirectional_connections

cognite/neat/_data_model/_snapshot.py

@@ -0,0 +1,133 @@
+import sys
+from datetime import datetime, timezone
+from typing import Any
+
+from pydantic import BaseModel, Field, field_serializer
+from pydantic_core.core_schema import FieldSerializationInfo
+
+from cognite.neat._client import NeatClient
+from cognite.neat._data_model.models.dms import (
+    ContainerReference,
+    ContainerRequest,
+    DataModelReference,
+    DataModelRequest,
+    NodeReference,
+    RequestSchema,
+    SpaceReference,
+    SpaceRequest,
+    ViewReference,
+    ViewRequest,
+)
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class SchemaSnapshot(BaseModel, extra="ignore"):
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    data_model: dict[DataModelReference, DataModelRequest] = Field(default_factory=dict)
+    views: dict[ViewReference, ViewRequest] = Field(default_factory=dict)
+    containers: dict[ContainerReference, ContainerRequest] = Field(default_factory=dict)
+    spaces: dict[SpaceReference, SpaceRequest] = Field(default_factory=dict)
+    node_types: dict[NodeReference, NodeReference] = Field(default_factory=dict)
+
+    @field_serializer("data_model", "views", "containers", "spaces", "node_types", mode="plain")
+    @classmethod
+    def make_hashable_keys(cls, value: dict, info: FieldSerializationInfo) -> dict[str, Any]:
+        output: dict[str, Any] = {}
+        for key, val in value.items():
+            dumped_value = val.model_dump(**vars(info))
+            output[str(key)] = dumped_value
+        return output
+
+    def merge(self, cdf: Self) -> Self:
+        """Merge another SchemaSnapshot into this one, prioritizing this snapshot's data."""
+        merged = self.model_copy(deep=True)
+
+        for model_ref, local_model in merged.data_model.items():
+            if model_ref not in cdf.data_model:
+                continue
+            cdf_model = cdf.data_model[model_ref]
+            if new_views := (set(cdf_model.views or []) - set(local_model.views or [])):
+                for view_ref in new_views:
+                    if cdf_view := cdf.views.get(view_ref):
+                        merged.views[view_ref] = cdf_view
+            # We append the local views at the end of the CDF views.
+            local_model.views = list(dict.fromkeys((cdf_model.views or []) + (local_model.views or [])).keys())
+
+        # Update local views with additional properties and implements from CDF views
+        for view_ref, view in merged.views.items():
+            if cdf_view := cdf.views.get(view_ref):
+                if cdf_only_containers := cdf_view.used_containers - set(view.used_containers):
+                    for cdf_only_container_ref in cdf_only_containers:
+                        if (
+                            cdf_container := cdf.containers.get(cdf_only_container_ref)
+                        ) and cdf_only_container_ref not in merged.containers:
+                            merged.containers[cdf_only_container_ref] = cdf_container
+
+                view.properties = {**cdf_view.properties, **view.properties}
+
+                # update implements
+                if cdf_view.implements:
+                    view.implements = list(dict.fromkeys(cdf_view.implements + (view.implements or [])).keys())
+
+        for container_ref, container in merged.containers.items():
+            if cdf_container := cdf.containers.get(container_ref):
+                container.properties = {**cdf_container.properties, **container.properties}
+
+        return merged
+
+    @classmethod
+    def fetch_cdf_data_model(cls, client: NeatClient, data_model: RequestSchema) -> Self:
+        """Fetch the latest data model, views, containers, and spaces from CDF based on the provided RequestSchema."""
+        now = datetime.now(timezone.utc)
+        space_ids = [space.as_reference() for space in data_model.spaces]
+        cdf_spaces = client.spaces.retrieve(space_ids)
+
+        container_refs = [c.as_reference() for c in data_model.containers]
+        cdf_containers = client.containers.retrieve(container_refs)
+
+        view_refs = [v.as_reference() for v in data_model.views]
+        cdf_views = client.views.retrieve(view_refs)
+
+        dm_ref = data_model.data_model.as_reference()
+        cdf_data_models = client.data_models.retrieve([dm_ref])
+
+        nodes = [node_type for view in cdf_views for node_type in view.node_types]
+        return cls(
+            timestamp=now,
+            data_model={dm.as_reference(): dm.as_request() for dm in cdf_data_models},
+            views={view.as_reference(): view.as_request() for view in cdf_views},
+            containers={container.as_reference(): container.as_request() for container in cdf_containers},
+            spaces={space.as_reference(): space.as_request() for space in cdf_spaces},
+            node_types={node: node for node in nodes},
+        )
+
+    @classmethod
+    def fetch_entire_cdf(cls, client: NeatClient) -> Self:
+        """Fetch the entire data model, views, containers, and spaces from CDF."""
+        now = datetime.now(timezone.utc)
+        all_views = client.views.list(
+            all_versions=True, include_global=True, include_inherited_properties=False, limit=None
+        )
+        nodes = [node_type for view in all_views for node_type in view.node_types]
+        return cls(
+            # TODO: spaces and data_models should be update after updating list methods for unlimited no
+            spaces={
+                response.as_reference(): response.as_request()
+                for response in client.spaces.list(include_global=True, limit=1000)
+            },
+            data_model={
+                response.as_reference(): response.as_request()
+                for response in client.data_models.list(all_versions=True, include_global=True, limit=1000)
+            },
+            views={response.as_reference(): response.as_request() for response in all_views},
+            containers={
+                response.as_reference(): response.as_request()
+                for response in client.containers.list(include_global=True, limit=None)
+            },
+            node_types={node_type: node_type for node_type in nodes},
+            timestamp=now,
+        )