cognite-neat 0.123.26__py3-none-any.whl → 1.0.22__py3-none-any.whl

cognite/neat/_data_model/_identifiers.py

@@ -0,0 +1,61 @@
+from pydantic import HttpUrl, RootModel, ValidationError
+
+from cognite.neat._utils.auxiliary import local_import
+
+
+class URI(RootModel[str]):
+    def __init__(self, value: str):
+        try:
+            # Use Pydantic's AnyUrl to validate the URI
+            _ = HttpUrl(value)
+        except ValidationError as e:
+            raise ValueError(f"Invalid URI: {value}") from e
+        super().__init__(value)
+
+    def __str__(self) -> str:
+        return self.root
+
+    def __repr__(self) -> str:
+        return f"URI({self.root!r})"
+
+    def as_rdflib_uriref(self):  # type: ignore[no-untyped-def]
+        # rdflib is an optional dependency, so import here
+        local_import("rdflib", "rdflib")
+        from rdflib import URIRef
+
+        return URIRef(self.root)
+
+
+class NameSpace(RootModel[str]):
+    def __init__(self, value: str):
+        try:
+            # Use Pydantic's AnyUrl to validate the URI
+            _ = HttpUrl(value)
+        except ValidationError as e:
+            raise ValueError(f"Invalid Namespace: {value}") from e
+        super().__init__(value)
+
+    def __str__(self) -> str:
+        return self.root
+
+    def __repr__(self) -> str:
+        return f"NameSpace({self.root!r})"
+
+    def term(self, name: str) -> URI:
+        # need to handle slices explicitly because of __getitem__ override
+        return URI(self.root + (name if isinstance(name, str) else ""))
+
+    def __getitem__(self, key: str) -> URI:  # type: ignore[override]
+        return self.term(key)
+
+    def __getattr__(self, name: str) -> URI:
+        if name.startswith("__"):  # ignore any special Python names!
+            raise AttributeError
+        return self.term(name)
+
+    def as_rdflib_namespace(self):  # type: ignore[no-untyped-def]
+        # rdflib is an optional dependency, so import here
+        local_import("rdflib", "rdflib")
+        from rdflib import Namespace
+
+        return Namespace(self.root)

cognite/neat/_data_model/_shared.py

@@ -0,0 +1,41 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from cognite.neat._data_model.deployer.data_classes import DeploymentResult
+from cognite.neat._issues import IssueList
+
+
+class OnSuccess(ABC):
+    """Abstract base class for post-activity success handlers."""
+
+    @abstractmethod
+    def run(self, data_model: Any) -> None:
+        """Execute the success handler on the data model."""
+        pass
+
+
+class OnSuccessIssuesChecker(OnSuccess, ABC):
+    """Abstract base class for post-activity success handlers that check for issues of the data model."""
+
+    def __init__(self) -> None:
+        self._issues = IssueList()
+        self._has_run = False
+
+    @property
+    def issues(self) -> IssueList:
+        if not self._has_run:
+            raise RuntimeError(f"{type(self).__name__} has not been run yet.")
+        return IssueList(self._issues)
+
+
+class OnSuccessResultProducer(OnSuccess, ABC):
+    """Abstract base class for post-activity success handlers that produce desired outcomes using the data model."""
+
+    def __init__(self) -> None:
+        self._results: DeploymentResult | None = None
+
+    @property
+    def result(self) -> DeploymentResult:
+        if self._results is None:
+            raise RuntimeError("SchemaDeployer has not been run yet.")
+        return self._results

cognite/neat/_data_model/_snapshot.py

@@ -0,0 +1,134 @@
+import sys
+from datetime import datetime, timezone
+from typing import Any
+
+from pydantic import BaseModel, Field, field_serializer
+from pydantic_core.core_schema import FieldSerializationInfo
+
+from cognite.neat._client import NeatClient
+from cognite.neat._data_model.models.dms import (
+    ContainerReference,
+    ContainerRequest,
+    DataModelReference,
+    DataModelRequest,
+    NodeReference,
+    RequestSchema,
+    SpaceReference,
+    SpaceRequest,
+    ViewReference,
+    ViewRequest,
+)
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class SchemaSnapshot(BaseModel, extra="ignore"):
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    data_model: dict[DataModelReference, DataModelRequest] = Field(default_factory=dict)
+    views: dict[ViewReference, ViewRequest] = Field(default_factory=dict)
+    containers: dict[ContainerReference, ContainerRequest] = Field(default_factory=dict)
+    spaces: dict[SpaceReference, SpaceRequest] = Field(default_factory=dict)
+    node_types: dict[NodeReference, NodeReference] = Field(default_factory=dict)
+
+    @field_serializer("data_model", "views", "containers", "spaces", "node_types", mode="plain")
+    @classmethod
+    def make_hashable_keys(cls, value: dict, info: FieldSerializationInfo) -> dict[str, Any]:
+        output: dict[str, Any] = {}
+        for key, val in value.items():
+            dumped_value = val.model_dump(**vars(info))
+            output[str(key)] = dumped_value
+        return output
+
+    def merge(self, cdf: Self) -> Self:
+        """Merge another SchemaSnapshot into this one, prioritizing this snapshot's data."""
+        merged = self.model_copy(deep=True)
+
+        for model_ref, local_model in merged.data_model.items():
+            if model_ref not in cdf.data_model:
+                continue
+            cdf_model = cdf.data_model[model_ref]
+            if new_views := (set(cdf_model.views or []) - set(local_model.views or [])):
+                for view_ref in new_views:
+                    if cdf_view := cdf.views.get(view_ref):
+                        merged.views[view_ref] = cdf_view
+            # We append the local views at the end of the CDF views.
+            local_model.views = list(dict.fromkeys((cdf_model.views or []) + (local_model.views or [])).keys())
+
+        # Update local views with additional properties and implements from CDF views
+        for view_ref, view in merged.views.items():
+            if cdf_view := cdf.views.get(view_ref):
+                if cdf_only_containers := cdf_view.used_containers - set(view.used_containers):
+                    for cdf_only_container_ref in cdf_only_containers:
+                        if (
+                            cdf_container := cdf.containers.get(cdf_only_container_ref)
+                        ) and cdf_only_container_ref not in merged.containers:
+                            merged.containers[cdf_only_container_ref] = cdf_container
+
+                view.properties = {**cdf_view.properties, **view.properties}
+
+                # update implements
+                if cdf_view.implements:
+                    view.implements = list(dict.fromkeys(cdf_view.implements + (view.implements or [])).keys())
+
+        for container_ref, container in merged.containers.items():
+            if cdf_container := cdf.containers.get(container_ref):
+                container.properties = {**cdf_container.properties, **container.properties}
+                container.constraints = {**(cdf_container.constraints or {}), **(container.constraints or {})} or None
+
+        return merged
+
+    @classmethod
+    def fetch_cdf_data_model(cls, client: NeatClient, data_model: RequestSchema) -> Self:
+        """Fetch the latest data model, views, containers, and spaces from CDF based on the provided RequestSchema."""
+        now = datetime.now(timezone.utc)
+        space_ids = [space.as_reference() for space in data_model.spaces]
+        cdf_spaces = client.spaces.retrieve(space_ids)
+
+        container_refs = [c.as_reference() for c in data_model.containers]
+        cdf_containers = client.containers.retrieve(container_refs)
+
+        view_refs = [v.as_reference() for v in data_model.views]
+        cdf_views = client.views.retrieve(view_refs)
+
+        dm_ref = data_model.data_model.as_reference()
+        cdf_data_models = client.data_models.retrieve([dm_ref])
+
+        nodes = [node_type for view in cdf_views for node_type in view.node_types]
+        return cls(
+            timestamp=now,
+            data_model={dm.as_reference(): dm.as_request() for dm in cdf_data_models},
+            views={view.as_reference(): view.as_request() for view in cdf_views},
+            containers={container.as_reference(): container.as_request() for container in cdf_containers},
+            spaces={space.as_reference(): space.as_request() for space in cdf_spaces},
+            node_types={node: node for node in nodes},
+        )
+
+    @classmethod
+    def fetch_entire_cdf(cls, client: NeatClient) -> Self:
+        """Fetch the entire data model, views, containers, and spaces from CDF."""
+        now = datetime.now(timezone.utc)
+        all_views = client.views.list(
+            all_versions=True, include_global=True, include_inherited_properties=False, limit=None
+        )
+        nodes = [node_type for view in all_views for node_type in view.node_types]
+        return cls(
+            # TODO: spaces and data_models should be update after updating list methods for unlimited no
+            spaces={
+                response.as_reference(): response.as_request()
+                for response in client.spaces.list(include_global=True, limit=1000)
+            },
+            data_model={
+                response.as_reference(): response.as_request()
+                for response in client.data_models.list(all_versions=True, include_global=True, limit=1000)
+            },
+            views={response.as_reference(): response.as_request() for response in all_views},
+            containers={
+                response.as_reference(): response.as_request()
+                for response in client.containers.list(include_global=True, limit=None)
+            },
+            node_types={node_type: node_type for node_type in nodes},
+            timestamp=now,
+        )

cognite/neat/_data_model/deployer/_differ.py

@@ -0,0 +1,140 @@
+from abc import ABC, abstractmethod
+from typing import Generic
+
+from cognite.neat._utils.useful_types import T_Item
+
+from .data_classes import (
+    AddedField,
+    ChangedField,
+    FieldChange,
+    FieldChanges,
+    RemovedField,
+    SeverityType,
+)
+
+
+class Differ(Generic[T_Item], ABC):
+    def __init__(self, parent_path: str | None = None) -> None:
+        self.parent_path = parent_path
+
+    def _get_path(self, field: str) -> str:
+        if self.parent_path:
+            return f"{self.parent_path}.{field}"
+        return field
+
+    def _diff_name_description(self, current: T_Item, new: T_Item, identifier: str | None = None) -> list[FieldChange]:
+        changes: list[FieldChange] = []
+        if hasattr(current, "name") and hasattr(new, "name"):
+            if current.name != new.name:
+                field_path = self._get_path(f"{identifier}.name" if identifier else "name")
+                changes.append(
+                    ChangedField(
+                        item_severity=SeverityType.SAFE,
+                        field_path=field_path,
+                        current_value=current.name,
+                        new_value=new.name,
+                    )
+                )
+        if hasattr(current, "description") and hasattr(new, "description"):
+            if current.description != new.description:
+                field_path = self._get_path(f"{identifier}.description" if identifier else "description")
+                changes.append(
+                    ChangedField(
+                        item_severity=SeverityType.SAFE,
+                        field_path=field_path,
+                        current_value=current.description,
+                        new_value=new.description,
+                    )
+                )
+        return changes
+
+
+class ItemDiffer(Differ[T_Item], ABC):
+    """A generic class for comparing two items of the same type and reporting the differences."""
+
+    @abstractmethod
+    def diff(self, current: T_Item, new: T_Item) -> list[FieldChange]:
+        """Compare two items and return a list of changes.
+
+        Args:
+            current: The resource as it is in CDF.
+            new: The resource as it is desired to be.
+
+        Returns:
+            A list of changes between the two resources.
+        """
+        raise NotImplementedError()
+
+
+class ObjectDiffer(Differ[T_Item], ABC):
+    @abstractmethod
+    def diff(self, current: T_Item, new: T_Item, identifier: str) -> list[FieldChange]:
+        """Compare two dict-like objects and return a list of changes.
+
+        Args:
+            current: The resource as it is in CDF.
+            new: The resource as it is desired to be.
+            identifier: The field used to identify individual items within the objects.
+
+        Returns:
+            A list of changes between the two resources.
+        """
+        raise NotImplementedError()
+
+
+def field_differences(
+    parent_path: str,
+    current: dict[str, T_Item] | None,
+    new: dict[str, T_Item] | None,
+    add_severity: SeverityType,
+    remove_severity: SeverityType,
+    differ: ObjectDiffer[T_Item],
+) -> list[FieldChange]:
+    """Diff two containers of items.
+
+    A container is for example the properties, constraints, or indexes of a container,
+    properties of a space, views of a data model, etc.
+
+    Args:
+        parent_path: The JSON path to the container being compared.
+        current: The items as they are in CDF.
+        new: The items as they are desired to be.
+        add_severity: The severity to assign to added items.
+        remove_severity: The severity to assign to removed items.
+        differ: The differ to use for comparing individual items.
+
+    """
+    changes: list[FieldChange] = []
+    current_map = current or {}
+    new_map = new or {}
+    current_keys = set(current_map.keys())
+    new_keys = set(new_map.keys())
+
+    for key in sorted(new_keys - current_keys):
+        item_path = f"{parent_path}.{key}"
+        changes.append(
+            AddedField(
+                item_severity=add_severity,
+                field_path=item_path,
+                new_value=new_map[key],
+            )
+        )
+
+    for key in sorted(current_keys - new_keys):
+        changes.append(
+            RemovedField(
+                item_severity=remove_severity,
+                field_path=f"{parent_path}.{key}",
+                current_value=current_map[key],
+            )
+        )
+
+    for key in sorted(current_keys & new_keys):
+        item_path = f"{parent_path}.{key}"
+        cdf_item = current_map[key]
+        desired_item = new_map[key]
+        diffs = differ.diff(cdf_item, desired_item, identifier=key)
+        if diffs:
+            changes.append(FieldChanges(field_path=item_path, changes=diffs))
+
+    return changes