cognite-neat 0.125.1__py3-none-any.whl → 0.126.1__py3-none-any.whl

cognite/neat/_data_model/models/dms/__init__.py

@@ -1,4 +1,10 @@
-from cognite.neat._data_model.models.dms._base import Resource, WriteableResource
+from cognite.neat._data_model.models.dms._base import (
+    APIResource,
+    BaseModelObject,
+    Resource,
+    T_Resource,
+    WriteableResource,
+)
 from cognite.neat._data_model.models.dms._constraints import (
     Constraint,
     ConstraintAdapter,
@@ -13,6 +19,7 @@ from cognite.neat._data_model.models.dms._container import (
     ContainerResponse,
 )
 from cognite.neat._data_model.models.dms._data_types import (
+    DMS_DATA_TYPES,
     BooleanProperty,
     DataType,
     DataTypeAdapter,
@@ -38,11 +45,13 @@ from cognite.neat._data_model.models.dms._indexes import BtreeIndex, Index, Inde
 from cognite.neat._data_model.models.dms._space import Space, SpaceRequest, SpaceResponse
 
 from ._data_model import DataModelRequest, DataModelResponse
+from ._http import DataModelBody, DataModelResource
 from ._references import (
     ContainerDirectReference,
     ContainerReference,
     DataModelReference,
     NodeReference,
+    SpaceReference,
     ViewDirectReference,
     ViewReference,
 )
@@ -60,6 +69,7 @@ from ._view_property import (
     ViewCorePropertyResponse,
     ViewPropertyDefinition,
     ViewRequestProperty,
+    ViewRequestPropertyAdapter,
     ViewResponseProperty,
 )
 from ._views import (
@@ -69,6 +79,9 @@ from ._views import (
 )
 
 __all__ = [
+    "DMS_DATA_TYPES",
+    "APIResource",
+    "BaseModelObject",
     "BooleanProperty",
     "BtreeIndex",
     "ConnectionPropertyDefinition",
@@ -82,8 +95,10 @@ __all__ = [
     "ContainerReference",
     "ContainerRequest",
     "ContainerResponse",
+    "DataModelBody",
     "DataModelReference",
     "DataModelRequest",
+    "DataModelResource",
     "DataModelResponse",
     "DataType",
     "DataTypeAdapter",
@@ -117,8 +132,10 @@ __all__ = [
     "SingleReverseDirectRelationPropertyRequest",
     "SingleReverseDirectRelationPropertyResponse",
     "Space",
+    "SpaceReference",
     "SpaceRequest",
     "SpaceResponse",
+    "T_Resource",
     "TextProperty",
     "TextProperty",
     "TimeseriesCDFExternalIdReference",
@@ -135,6 +152,7 @@ __all__ = [
     "ViewReference",
     "ViewRequest",
     "ViewRequestProperty",
+    "ViewRequestPropertyAdapter",
     "ViewResponse",
     "ViewResponseProperty",
     "WriteableResource",

cognite/neat/_data_model/models/dms/_base.py

@@ -1,14 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import Generic, TypeVar
 
-from pydantic import BaseModel
-from pydantic.alias_generators import to_camel
-
-
-class BaseModelObject(BaseModel, alias_generator=to_camel, extra="ignore"):
-    """Base class for all object. This includes resources and nested objects."""
-
-    ...
+from cognite.neat._utils.useful_types import BaseModelObject, T_Reference
 
 
 class Resource(BaseModelObject):
@@ -21,7 +14,18 @@ T_Resource = TypeVar("T_Resource", bound=Resource)
 
 
 class WriteableResource(Resource, Generic[T_Resource], ABC):
+    """Base class for all writeable data modeling resources."""
+
     @abstractmethod
     def as_request(self) -> T_Resource:
         """Convert the response model to a request model by removing read-only fields."""
         raise NotImplementedError()
+
+
+class APIResource(Generic[T_Reference], ABC):
+    """Base class for all API data modeling resources."""
+
+    @abstractmethod
+    def as_reference(self) -> T_Reference:
+        """Convert the resource to a reference object (identifier)."""
+        raise NotImplementedError()

cognite/neat/_data_model/models/dms/_constants.py

@@ -1,7 +1,7 @@
 SPACE_FORMAT_PATTERN = r"^[a-zA-Z][a-zA-Z0-9_-]{0,41}[a-zA-Z0-9]?$"
 DM_EXTERNAL_ID_PATTERN = r"^[a-zA-Z]([a-zA-Z0-9_]{0,253}[a-zA-Z0-9])?$"
 CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN = r"^[a-zA-Z0-9][a-zA-Z0-9_-]{0,253}[a-zA-Z0-9]?$"
-INSTANCE_ID_PATTERN = r"^[^\\x00]{1,256}$"
+INSTANCE_ID_PATTERN = r"^[^\x00]{1,256}$"
 ENUM_VALUE_IDENTIFIER_PATTERN = r"^[_A-Za-z][_0-9A-Za-z]{0,127}$"
 DM_VERSION_PATTERN = r"^[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])?$"
 FORBIDDEN_ENUM_VALUES = frozenset({"true", "false", "null"})

cognite/neat/_data_model/models/dms/_constraints.py

@@ -3,7 +3,8 @@ from typing import Annotated, Literal
 
 from pydantic import Field, TypeAdapter
 
-from ._base import BaseModelObject
+from cognite.neat._utils.useful_types import BaseModelObject
+
 from ._references import ContainerReference
 from ._types import Bool
 

cognite/neat/_data_model/models/dms/_container.py

@@ -6,8 +6,9 @@ from pydantic import Field, field_validator
 from pydantic_core.core_schema import ValidationInfo
 
 from cognite.neat._utils.text import humanize_collection
+from cognite.neat._utils.useful_types import BaseModelObject
 
-from ._base import BaseModelObject, Resource, WriteableResource
+from ._base import APIResource, Resource, WriteableResource
 from ._constants import (
     CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN,
     DM_EXTERNAL_ID_PATTERN,
@@ -36,7 +37,7 @@ class ContainerPropertyDefinition(BaseModelObject):
         default=None,
         description="Increment the property based on its highest current value (max value).",
     )
-    default_value: str | int | bool | dict[str, Any] | None = Field(
+    default_value: str | int | float | bool | dict[str, Any] | None = Field(
         default=None,
         description="Default value to use when you do not specify a value for the property.",
     )
@@ -53,7 +54,7 @@ class ContainerPropertyDefinition(BaseModelObject):
     type: DataType = Field(description="The type of data you can store in this property.")
 
 
-class Container(Resource, ABC):
+class Container(Resource, APIResource[ContainerReference], ABC):
     space: str = Field(
         description="The workspace for the container, a unique identifier for the space.",
         min_length=1,
@@ -82,7 +83,6 @@ class Container(Resource, ABC):
     )
     properties: dict[str, ContainerPropertyDefinition] = Field(
         description="Set of properties to apply to the container.",
-        min_length=1,
     )
     constraints: dict[str, Constraint] | None = Field(
         default=None,
@@ -136,7 +136,7 @@ class Container(Resource, ABC):
     def as_reference(self) -> ContainerReference:
         return ContainerReference(
             space=self.space,
-            externalId=self.external_id,
+            external_id=self.external_id,
         )
 
 

cognite/neat/_data_model/models/dms/_data_model.py

@@ -2,7 +2,7 @@ from abc import ABC
 
 from pydantic import Field
 
-from ._base import Resource, WriteableResource
+from ._base import APIResource, Resource, WriteableResource
 from ._constants import (
     DM_EXTERNAL_ID_PATTERN,
     DM_VERSION_PATTERN,
@@ -11,7 +11,7 @@ from ._constants import (
 from ._references import DataModelReference, ViewReference
 
 
-class DataModel(Resource, ABC):
+class DataModel(Resource, APIResource[DataModelReference], ABC):
     """Cognite Data Model resource.
 
     Data models group and structure views into reusable collections.
@@ -55,7 +55,7 @@ class DataModel(Resource, ABC):
     def as_reference(self) -> DataModelReference:
         return DataModelReference(
             space=self.space,
-            externalId=self.external_id,
+            external_id=self.external_id,
             version=self.version,
         )
 

cognite/neat/_data_model/models/dms/_data_types.py

@@ -1,10 +1,13 @@
 import re
 from abc import ABC
+from collections.abc import Mapping
 from typing import Annotated, Literal
 
 from pydantic import Field, TypeAdapter, field_validator
 
-from ._base import BaseModelObject
+from cognite.neat._utils.auxiliary import get_concrete_subclasses
+from cognite.neat._utils.useful_types import BaseModelObject
+
 from ._constants import ENUM_VALUE_IDENTIFIER_PATTERN, FORBIDDEN_ENUM_VALUES, INSTANCE_ID_PATTERN
 from ._references import ContainerReference
 
@@ -181,3 +184,7 @@ DataType = Annotated[
 ]
 
 DataTypeAdapter: TypeAdapter[DataType] = TypeAdapter(DataType)
+
+DMS_DATA_TYPES: Mapping[str, type[PropertyTypeDefinition]] = {
+    cls_.model_fields["type"].default: cls_ for cls_ in get_concrete_subclasses(PropertyTypeDefinition)
+}

cognite/neat/_data_model/models/dms/_http.py

@@ -0,0 +1,18 @@
+from typing import TypeAlias, TypeVar
+
+from cognite.neat._utils.http_client import ItemBody
+from cognite.neat._utils.useful_types import ReferenceObject
+
+from ._container import ContainerRequest
+from ._data_model import DataModelRequest
+from ._space import SpaceRequest
+from ._views import ViewRequest
+
+DataModelResource: TypeAlias = SpaceRequest | DataModelRequest | ViewRequest | ContainerRequest
+
+T_DataModelResource = TypeVar("T_DataModelResource", bound=DataModelResource)
+
+
+class DataModelBody(ItemBody[ReferenceObject, DataModelResource]):
+    def as_ids(self) -> list[ReferenceObject]:
+        return [item.as_reference() for item in self.items]

cognite/neat/_data_model/models/dms/_indexes.py

@@ -3,7 +3,8 @@ from typing import Annotated, Literal
 
 from pydantic import Field, TypeAdapter
 
-from ._base import BaseModelObject
+from cognite.neat._utils.useful_types import BaseModelObject
+
 from ._types import Bool
 
 

cognite/neat/_data_model/models/dms/_references.py

@@ -2,7 +2,8 @@ from typing import Literal
 
 from pydantic import Field
 
-from ._base import BaseModelObject
+from cognite.neat._utils.useful_types import ReferenceObject
+
 from ._constants import (
     CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN,
     DM_EXTERNAL_ID_PATTERN,
@@ -12,11 +13,17 @@ from ._constants import (
 )
 
 
-class ReferenceObject(BaseModelObject, frozen=True): ...
+class SpaceReference(ReferenceObject):
+    space: str = Field(
+        description="Id of the space.",
+        min_length=1,
+        max_length=43,
+        pattern=SPACE_FORMAT_PATTERN,
+    )
 
 
 class ContainerReference(ReferenceObject):
-    type: Literal["container"] = "container"
+    type: Literal["container"] = Field("container", exclude=True)
     space: str = Field(
         description="Id of the space hosting (containing) the container.",
         min_length=1,
@@ -30,9 +37,12 @@ class ContainerReference(ReferenceObject):
         pattern=DM_EXTERNAL_ID_PATTERN,
     )
 
+    def __str__(self) -> str:
+        return f"{self.space}:{self.external_id}"
+
 
 class ViewReference(ReferenceObject):
-    type: Literal["view"] = "view"
+    type: Literal["view"] = Field("view", exclude=True)
     space: str = Field(
         description="Id of the space that the view belongs to.",
         min_length=1,
@@ -51,6 +61,9 @@ class ViewReference(ReferenceObject):
         pattern=DM_VERSION_PATTERN,
     )
 
+    def __str__(self) -> str:
+        return f"{self.space}:{self.external_id}(version={self.version})"
+
 
 class DataModelReference(ReferenceObject):
     space: str = Field(

cognite/neat/_data_model/models/dms/_space.py

@@ -2,13 +2,14 @@ from abc import ABC
 
 from pydantic import Field, field_validator
 
+from cognite.neat._data_model.models.dms._references import SpaceReference
 from cognite.neat.v0.core._utils.text import humanize_collection
 
-from ._base import WriteableResource
+from ._base import APIResource, Resource, WriteableResource
 from ._constants import FORBIDDEN_SPACES, SPACE_FORMAT_PATTERN
 
 
-class Space(WriteableResource["SpaceRequest"], ABC):
+class Space(Resource, APIResource[SpaceReference], ABC):
     space: str = Field(
         description="The Space identifier (id).",
         min_length=1,
@@ -25,11 +26,14 @@ class Space(WriteableResource["SpaceRequest"], ABC):
             raise ValueError(f"{val!r} is a reserved space. Reserved Spaces: {humanize_collection(FORBIDDEN_SPACES)}")
         return val
 
-    def as_request(self) -> "SpaceRequest":
-        return SpaceRequest.model_validate(self.model_dump(by_alias=True))
+    def as_reference(self) -> SpaceReference:
+        return SpaceReference(space=self.space)
+
+
+class SpaceRequest(Space): ...
 
 
-class SpaceResponse(Space):
+class SpaceResponse(Space, WriteableResource[Space]):
     created_time: int = Field(
         description="When the space was created. The number of milliseconds since 00:00:00 Thursday, 1 January 1970, "
         "Coordinated Universal Time (UTC), minus leap seconds."
@@ -40,5 +44,5 @@ class SpaceResponse(Space):
     )
     is_global: bool = Field(description="Whether the space is a global space.")
 
-
-class SpaceRequest(Space): ...
+    def as_request(self) -> "SpaceRequest":
+        return SpaceRequest.model_validate(self.model_dump(by_alias=True))

cognite/neat/_data_model/models/dms/_view_property.py

@@ -3,7 +3,9 @@ from typing import Annotated, Literal
 
 from pydantic import Field, Json, TypeAdapter
 
-from ._base import BaseModelObject, Resource, WriteableResource
+from cognite.neat._utils.useful_types import BaseModelObject
+
+from ._base import Resource, WriteableResource
 from ._constants import CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN
 from ._data_types import DataType
 from ._references import ContainerDirectReference, ContainerReference, NodeReference, ViewDirectReference, ViewReference
@@ -49,6 +51,7 @@ class ViewCorePropertyRequest(ViewCoreProperty): ...
 
 class ConstraintOrIndexState(BaseModelObject):
     nullability: Literal["current", "pending", "failed"] | None = Field(
+        None,
         description="""For properties that have isNullable set to false, this field describes the validity of the
 not-null constraint. It is not specified for nullable properties.
 
@@ -58,7 +61,7 @@ Possible values are:
           existing nulls was made non-nullable. New null values will still be rejected.
 "current": The constraint is satisfied; all values in the property are not null.
 "pending": The constraint validity has not yet been computed.
-        """
+        """,
     )
 
 
@@ -147,7 +150,7 @@ class SingleReverseDirectRelationPropertyResponse(
     ReverseDirectRelationProperty, WriteableResource[SingleReverseDirectRelationPropertyRequest]
 ):
     connection_type: Literal["single_reverse_direct_relation"] = "single_reverse_direct_relation"
-    target_list: bool = Field(
+    targets_list: bool = Field(
         description="Whether or not this reverse direct relation targets a list of direct relations.",
     )
 
@@ -159,7 +162,7 @@ class MultiReverseDirectRelationPropertyResponse(
     ReverseDirectRelationProperty, WriteableResource[MultiReverseDirectRelationPropertyRequest]
 ):
     connection_type: Literal["multi_reverse_direct_relation"] = "multi_reverse_direct_relation"
-    target_list: bool = Field(
+    targets_list: bool = Field(
         description="Whether or not this reverse direct relation targets a list of direct relations.",
     )
 

cognite/neat/_data_model/models/dms/_views.py

@@ -2,11 +2,11 @@ import re
 from abc import ABC
 from typing import Literal, TypeVar
 
-from pydantic import Field, Json, field_validator, model_validator
+from pydantic import Field, JsonValue, field_validator, model_validator
 
 from cognite.neat._utils.text import humanize_collection
 
-from ._base import Resource, WriteableResource
+from ._base import APIResource, Resource, WriteableResource
 from ._constants import (
     CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN,
     DM_EXTERNAL_ID_PATTERN,
@@ -15,8 +15,9 @@ from ._constants import (
     FORBIDDEN_CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER,
     SPACE_FORMAT_PATTERN,
 )
-from ._references import ContainerReference, ViewReference
+from ._references import ContainerReference, NodeReference, ViewReference
 from ._view_property import (
+    EdgeProperty,
     ViewRequestProperty,
     ViewResponseProperty,
 )
@@ -24,7 +25,7 @@ from ._view_property import (
 KEY_PATTERN = re.compile(CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN)
 
 
-class View(Resource, ABC):
+class View(Resource, APIResource[ViewReference], ABC):
     space: str = Field(
         description="Id of the space that the view belongs to.",
         min_length=1,
@@ -52,7 +53,7 @@ class View(Resource, ABC):
         description="Description of the view.",
         max_length=1024,
     )
-    filter: dict[str, Json] | None = Field(
+    filter: dict[str, JsonValue] | None = Field(
         default=None,
         description="A filter Domain Specific Language (DSL) used to create advanced filter queries.",
     )
@@ -62,7 +63,7 @@ class View(Resource, ABC):
     )
 
     def as_reference(self) -> ViewReference:
-        return ViewReference(space=self.space, externalId=self.external_id, version=self.version)
+        return ViewReference(space=self.space, external_id=self.external_id, version=self.version)
 
     @model_validator(mode="before")
     def set_connection_type_on_primary_properties(cls, data: dict) -> dict:
@@ -134,6 +135,15 @@ class ViewResponse(View, WriteableResource[ViewRequest]):
         """Validate properties Identifier"""
         return _validate_properties_keys(val)
 
+    @property
+    def node_types(self) -> list[NodeReference]:
+        """Get all node types referenced by this view."""
+        nodes_refs: set[NodeReference] = set()
+        for prop in self.properties.values():
+            if isinstance(prop, EdgeProperty):
+                nodes_refs.add(prop.type)
+        return list(nodes_refs)
+
     def as_request(self) -> ViewRequest:
         dumped = self.model_dump(by_alias=True, exclude={"properties"})
         dumped["properties"] = {

cognite/neat/_data_model/validation/_base.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+from typing import ClassVar
+
+from cognite.neat._issues import ConsistencyError, Recommendation
+
+
+class DataModelValidator(ABC):
+    """Assessors for fundamental data model principles."""
+
+    code: ClassVar[str]
+
+    @abstractmethod
+    def run(self) -> list[ConsistencyError] | list[Recommendation]:
+        """Execute the success handler on the data model."""
+        # do something with data model
+        pass

cognite/neat/_data_model/validation/dms/__init__.py

@@ -0,0 +1,9 @@
+from ._orchestrator import DmsDataModelValidation
+from ._validators import UndefinedConnectionEndNodeTypes, VersionSpaceInconsistency, ViewsWithoutProperties
+
+__all__ = [
+    "DmsDataModelValidation",
+    "UndefinedConnectionEndNodeTypes",
+    "VersionSpaceInconsistency",
+    "ViewsWithoutProperties",
+]

cognite/neat/_data_model/validation/dms/_orchestrator.py

@@ -0,0 +1,68 @@
+from cognite.neat._client import NeatClient
+from cognite.neat._data_model._analysis import DataModelAnalysis
+from cognite.neat._data_model._shared import OnSuccessIssuesChecker
+from cognite.neat._data_model.models.dms._references import ViewReference
+from cognite.neat._data_model.models.dms._schema import RequestSchema
+from cognite.neat._data_model.models.dms._views import ViewRequest
+from cognite.neat._data_model.validation._base import DataModelValidator
+
+from ._validators import UndefinedConnectionEndNodeTypes, VersionSpaceInconsistency, ViewsWithoutProperties
+
+
+class DmsDataModelValidation(OnSuccessIssuesChecker):
+    """Placeholder for DMS Quality Assessment functionality."""
+
+    def __init__(
+        self, client: NeatClient | None = None, codes: list[str] | None = None, modus_operandi: str | None = None
+    ) -> None:
+        super().__init__(client)
+        self._codes = codes or ["all"]
+        self._modus_operandi = modus_operandi  # will be used later to trigger how validators will behave
+
+    def run(self, data_model: RequestSchema) -> None:
+        """Run quality assessment on the DMS data model."""
+
+        # Helper wrangled data model components
+        analysis = DataModelAnalysis(data_model)
+        local_views_by_reference = analysis.view_by_reference(include_inherited_properties=True)
+        local_connection_end_node_types = analysis.connection_end_node_types
+        cdf_views_by_reference = self._cdf_view_by_reference(
+            list(analysis.referenced_views(include_connection_end_node_types=True)),
+            include_inherited_properties=True,
+        )
+
+        validators: list[DataModelValidator] = [
+            ViewsWithoutProperties(
+                local_views_by_reference=local_views_by_reference,
+                cdf_views_by_reference=cdf_views_by_reference,
+            ),
+            UndefinedConnectionEndNodeTypes(
+                local_connection_end_node_types=local_connection_end_node_types,
+                local_views_by_reference=local_views_by_reference,
+                cdf_views_by_reference=cdf_views_by_reference,
+            ),
+            VersionSpaceInconsistency(
+                data_model_reference=data_model.data_model.as_reference(),
+                view_references=list(local_views_by_reference.keys()),
+            ),
+        ]
+
+        for validator in validators:
+            if "all" in self._codes or validator.code in self._codes:
+                self._issues.extend(validator.run())
+
+        self._has_run = True
+
+    def _cdf_view_by_reference(
+        self, views: list[ViewReference], include_inherited_properties: bool = True
+    ) -> dict[ViewReference, ViewRequest]:
+        """Fetch view definition from CDF."""
+
+        if not self._client:
+            return {}
+        return {
+            response.as_reference(): response.as_request()
+            for response in self._client.views.retrieve(
+                views, include_inherited_properties=include_inherited_properties
+            )
+        }