cognite-neat 0.123.26__py3-none-any.whl → 1.0.22__py3-none-any.whl

cognite/neat/_data_model/models/dms/_references.py

@@ -0,0 +1,141 @@
+from typing import Literal
+
+from pydantic import Field
+
+from cognite.neat._utils.useful_types import ReferenceObject
+
+from ._constants import (
+    CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN,
+    DM_EXTERNAL_ID_PATTERN,
+    DM_VERSION_PATTERN,
+    INSTANCE_ID_PATTERN,
+    SPACE_FORMAT_PATTERN,
+)
+
+
+class SpaceReference(ReferenceObject):
+    space: str = Field(
+        description="Id of the space.",
+        min_length=1,
+        max_length=43,
+        pattern=SPACE_FORMAT_PATTERN,
+    )
+
+
+class ContainerReference(ReferenceObject):
+    type: Literal["container"] = Field("container", exclude=True)
+    space: str = Field(
+        description="Id of the space hosting (containing) the container.",
+        min_length=1,
+        max_length=43,
+        pattern=SPACE_FORMAT_PATTERN,
+    )
+    external_id: str = Field(
+        description="External-id of the container.",
+        min_length=1,
+        max_length=255,
+        pattern=DM_EXTERNAL_ID_PATTERN,
+    )
+
+    def __str__(self) -> str:
+        return f"{self.space}:{self.external_id}"
+
+
+class ViewReference(ReferenceObject):
+    type: Literal["view"] = Field("view", exclude=True)
+    space: str = Field(
+        description="Id of the space that the view belongs to.",
+        min_length=1,
+        max_length=43,
+        pattern=SPACE_FORMAT_PATTERN,
+    )
+    external_id: str = Field(
+        description="External-id of the view.",
+        min_length=1,
+        max_length=255,
+        pattern=DM_EXTERNAL_ID_PATTERN,
+    )
+    version: str = Field(
+        description="Version of the view.",
+        max_length=43,
+        pattern=DM_VERSION_PATTERN,
+    )
+
+    def __str__(self) -> str:
+        return f"{self.space}:{self.external_id}(version={self.version})"
+
+
+class DataModelReference(ReferenceObject):
+    space: str = Field(
+        description="Id of the space that the data model belongs to.",
+        min_length=1,
+        max_length=43,
+        pattern=SPACE_FORMAT_PATTERN,
+    )
+    external_id: str = Field(
+        description="External-id of the data model.",
+        min_length=1,
+        max_length=255,
+        pattern=DM_EXTERNAL_ID_PATTERN,
+    )
+    version: str = Field(
+        description="Version of the data model.",
+        max_length=43,
+        pattern=DM_VERSION_PATTERN,
+    )
+
+    def __str__(self) -> str:
+        return f"{self.space}:{self.external_id}(version={self.version})"
+
+
+class NodeReference(ReferenceObject):
+    space: str = Field(
+        description="Id of the space hosting (containing) the node.",
+        min_length=1,
+        max_length=43,
+        pattern=SPACE_FORMAT_PATTERN,
+    )
+    external_id: str = Field(
+        description="External-id of the node.",
+        min_length=1,
+        max_length=255,
+        pattern=INSTANCE_ID_PATTERN,
+    )
+
+
+class ContainerDirectReference(ReferenceObject):
+    source: ContainerReference = Field(
+        description="Reference to the container from where this relation is inherited.",
+    )
+    identifier: str = Field(
+        description="Identifier of the relation in the source container.",
+        min_length=1,
+        max_length=255,
+        pattern=CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN,
+    )
+
+    def __str__(self) -> str:
+        return f"{self.source!s}.{self.identifier}"
+
+
+class ViewDirectReference(ReferenceObject):
+    source: ViewReference = Field(
+        description="Reference to the view from where this relation is inherited.",
+    )
+    identifier: str = Field(
+        description="Identifier of the relation in the source view.",
+        min_length=1,
+        max_length=255,
+        pattern=CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN,
+    )
+
+    def __str__(self) -> str:
+        return f"{self.source!s}.{self.identifier}"
+
+
+class ContainerIndexReference(ContainerReference):
+    identifier: str
+
+
+class ContainerConstraintReference(ContainerReference):
+    identifier: str

cognite/neat/_data_model/models/dms/_schema.py

@@ -0,0 +1,18 @@
+from pydantic import Field
+
+from ._base import Resource
+from ._container import ContainerRequest
+from ._data_model import DataModelRequest
+from ._references import NodeReference
+from ._space import SpaceRequest
+from ._views import ViewRequest
+
+
+class RequestSchema(Resource):
+    """Represents a schema for creating or updating a data model in DMS."""
+
+    data_model: DataModelRequest
+    views: list[ViewRequest] = Field(default_factory=list)
+    containers: list[ContainerRequest] = Field(default_factory=list)
+    spaces: list[SpaceRequest] = Field(default_factory=list)
+    node_types: list[NodeReference] = Field(default_factory=list)

cognite/neat/_data_model/models/dms/_space.py

@@ -0,0 +1,48 @@
+from abc import ABC
+
+from pydantic import Field, field_validator
+
+from cognite.neat._data_model.models.dms._references import SpaceReference
+from cognite.neat._utils.text import humanize_collection
+
+from ._base import APIResource, Resource, WriteableResource
+from ._constants import FORBIDDEN_SPACES, SPACE_FORMAT_PATTERN
+
+
+class Space(Resource, APIResource[SpaceReference], ABC):
+    space: str = Field(
+        description="The Space identifier (id).",
+        min_length=1,
+        max_length=43,
+        pattern=SPACE_FORMAT_PATTERN,
+    )
+    name: str | None = Field(None, description="Name of the space.", max_length=255)
+    description: str | None = Field(None, description="The description of the space.", max_length=1024)
+
+    @field_validator("space")
+    def check_forbidden_space_value(cls, val: str) -> str:
+        """Check the space name not present in forbidden set"""
+        if val in FORBIDDEN_SPACES:
+            raise ValueError(f"{val!r} is a reserved space. Reserved Spaces: {humanize_collection(FORBIDDEN_SPACES)}")
+        return val
+
+    def as_reference(self) -> SpaceReference:
+        return SpaceReference(space=self.space)
+
+
+class SpaceRequest(Space): ...
+
+
+class SpaceResponse(Space, WriteableResource[Space]):
+    created_time: int = Field(
+        description="When the space was created. The number of milliseconds since 00:00:00 Thursday, 1 January 1970, "
+        "Coordinated Universal Time (UTC), minus leap seconds."
+    )
+    last_updated_time: int = Field(
+        description="When the space was last updated. The number of milliseconds since 00:00:00 Thursday, "
+        "1 January 1970, Coordinated Universal Time (UTC), minus leap seconds."
+    )
+    is_global: bool = Field(description="Whether the space is a global space.")
+
+    def as_request(self) -> "SpaceRequest":
+        return SpaceRequest.model_validate(self.model_dump(by_alias=True))

cognite/neat/_data_model/models/dms/_types.py

@@ -0,0 +1,17 @@
+from typing import Annotated, Any
+
+from pydantic import BeforeValidator
+
+
+def str_as_bool(value: Any) -> Any:
+    if isinstance(value, str):
+        val = value.lower()
+        if val in {"true", "1", "yes"}:
+            return True
+        if val in {"false", "0", "no"}:
+            return False
+    # All other cases are handled by Pydantic's built-in bool validator
+    return value
+
+
+Bool = Annotated[bool, BeforeValidator(str_as_bool, str)]

cognite/neat/_data_model/models/dms/_view_filter.py

@@ -0,0 +1,310 @@
+from abc import ABC
+from typing import Annotated, Any, Literal, TypeAlias, get_args
+
+from pydantic import BeforeValidator, Field, JsonValue, TypeAdapter, model_serializer
+from pydantic_core.core_schema import FieldSerializationInfo
+
+from cognite.neat._utils.text import humanize_collection
+from cognite.neat._utils.useful_types import BaseModelObject
+
+from ._references import ContainerReference, NodeReference, ViewReference
+
+# Base classes and helpers
+
+
+class Parameter(BaseModelObject):
+    parameter: str
+
+
+class FilterDataDefinition(BaseModelObject, ABC):
+    """Base class for filter data models."""
+
+    # This is an internal field used for discriminating between filter types. It is not part of the actual
+    # data sent to or received from the API. See the _move_filter_key function for more details.
+    filter_type: str = Field(..., exclude=True)
+
+
+class PropertyReference(FilterDataDefinition, ABC):
+    """Represents the property path in filters."""
+
+    property: list[str] = Field(..., min_length=2, max_length=3)
+
+
+## Leaf filters that follows the standard pattern
+
+
+class EqualsFilterData(PropertyReference):
+    filter_type: Literal["equals"] = Field("equals", exclude=True)
+    value: JsonValue | PropertyReference
+
+
+class InFilterData(PropertyReference):
+    filter_type: Literal["in"] = Field("in", exclude=True)
+    values: list[JsonValue] | PropertyReference
+
+
+class RangeFilterData(PropertyReference):
+    filter_type: Literal["range"] = Field("range", exclude=True)
+    gt: str | int | float | PropertyReference | None = None
+    gte: str | int | float | PropertyReference | None = None
+    lt: str | int | float | PropertyReference | None = None
+    lte: str | int | float | PropertyReference | None = None
+
+
+class PrefixFilterData(PropertyReference):
+    filter_type: Literal["prefix"] = Field("prefix", exclude=True)
+    value: str | Parameter
+
+
+class ExistsFilterData(PropertyReference):
+    filter_type: Literal["exists"] = Field("exists", exclude=True)
+
+
+class ContainsAnyFilterData(PropertyReference):
+    filter_type: Literal["containsAny"] = Field("containsAny", exclude=True)
+    values: list[JsonValue] | PropertyReference
+
+
+class ContainsAllFilterData(PropertyReference):
+    filter_type: Literal["containsAll"] = Field("containsAll", exclude=True)
+    values: list[JsonValue] | PropertyReference
+
+
+class MatchAllFilterData(FilterDataDefinition):
+    filter_type: Literal["matchAll"] = Field("matchAll", exclude=True)
+
+
+class NestedFilterData(FilterDataDefinition):
+    filter_type: Literal["nested"] = Field("nested", exclude=True)
+    scope: list[str] = Field(..., min_length=1, max_length=3)
+    filter: "Filter"
+
+
+class OverlapsFilterData(PropertyReference):
+    filter_type: Literal["overlaps"] = Field("overlaps", exclude=True)
+    start_property: list[str] = Field(..., min_length=1, max_length=3)
+    end_property: list[str] = Field(..., min_length=1, max_length=3)
+    gt: str | int | float | PropertyReference | None = None
+    gte: str | int | float | PropertyReference | None = None
+    lt: str | int | float | PropertyReference | None = None
+    lte: str | int | float | PropertyReference | None = None
+
+
+class ListFilterDataDefinition(FilterDataDefinition, ABC):
+    """Base class for filters that operate on lists of values."""
+
+    data: list[Any]
+
+
+## Leaf filters with custom serialization logic due to creativity in the API design
+
+
+class HasDataFilter(ListFilterDataDefinition):
+    filter_type: Literal["hasData"] = Field("hasData", exclude=True)
+    data: list[ViewReference | ContainerReference]
+
+    # MyPy complains about thet signature of the method here, even though its compatible with the pydantic source code.
+    # And tests are passing fine.
+    @model_serializer(mode="plain")  # type: ignore[type-var]
+    def serialize_model(self, info: FieldSerializationInfo) -> list[dict[str, Any]]:
+        output: list[dict[str, Any]] = []
+        for item in self.data:
+            item_dict = item.model_dump(**vars(info))
+            if isinstance(item, ViewReference):
+                item_dict["type"] = "view"
+            elif isinstance(item, ContainerReference):
+                item_dict["type"] = "container"
+            output.append(item_dict)
+        return output
+
+
+class InstanceReferencesFilterData(ListFilterDataDefinition):
+    filter_type: Literal["instanceReferences"] = Field("instanceReferences", exclude=True)
+    data: list[NodeReference]
+
+    # MyPy complains about thet signature of the method here, even though its compatible with the pydantic source code.
+    # And tests are passing fine.
+    @model_serializer(mode="plain")  # type: ignore[type-var]
+    def serialize_model(self, info: FieldSerializationInfo) -> list[dict[str, Any]]:
+        return [item.model_dump(**vars(info)) for item in self.data]
+
+
+## Logical filters combining other filters
+
+
+class AndFilter(ListFilterDataDefinition):
+    filter_type: Literal["and"] = Field("and", exclude=True)
+    data: "list[Filter]"
+
+    # MyPy complains about thet signature of the method here, even though its compatible with the pydantic source code.
+    # And tests are passing fine.
+    @model_serializer(mode="plain")  # type: ignore[type-var]
+    def serialize_model(self, info: FieldSerializationInfo) -> list[dict[str, Any]]:
+        return [FilterAdapter.dump_python(item, **vars(info)) for item in self.data]
+
+
+class OrFilter(ListFilterDataDefinition):
+    filter_type: Literal["or"] = Field("or", exclude=True)
+    data: "list[Filter]"
+
+    # MyPy complains about thet signature of the method here, even though its compatible with the pydantic source code.
+    # And tests are passing fine.
+    @model_serializer(mode="plain")  # type: ignore[type-var]
+    def serialize_model(self, info: FieldSerializationInfo) -> list[dict[str, Any]]:
+        return [FilterAdapter.dump_python(item, **vars(info)) for item in self.data]
+
+
+class NotFilter(FilterDataDefinition):
+    filter_type: Literal["not"] = Field("not", exclude=True)
+    data: "Filter"
+
+    # MyPy complains about thet signature of the method here, even though its compatible with the pydantic source code.
+    # And tests are passing fine.
+    @model_serializer(mode="plain")  # type: ignore[type-var]
+    def serialize_model(self, info: FieldSerializationInfo) -> dict[str, Any]:
+        return FilterAdapter.dump_python(self.data, **vars(info))
+
+
+FilterData = Annotated[
+    EqualsFilterData
+    | PrefixFilterData
+    | InFilterData
+    | RangeFilterData
+    | ExistsFilterData
+    | ContainsAnyFilterData
+    | ContainsAllFilterData
+    | MatchAllFilterData
+    | NestedFilterData
+    | OverlapsFilterData
+    | HasDataFilter
+    | InstanceReferencesFilterData
+    | AndFilter
+    | OrFilter
+    | NotFilter,
+    Field(discriminator="filter_type"),
+]
+
+
+FilterTypes: TypeAlias = Literal[
+    "equals",
+    "prefix",
+    "in",
+    "range",
+    "exists",
+    "containsAny",
+    "containsAll",
+    "matchAll",
+    "nested",
+    "overlaps",
+    "hasData",
+    "instanceReferences",
+    "and",
+    "or",
+    "not",
+]
+
+LegacyFilterTypes: TypeAlias = Literal["invalid"]
+
+AVAILABLE_FILTERS: frozenset[str] = frozenset(get_args(FilterTypes))
+LEGACY_FILTERS: frozenset[str] = frozenset(get_args(LegacyFilterTypes))
+
+
+def _move_filter_key(value: Any) -> Any:
+    """The DMS API filters have an unusual structure.
+
+    It has the filter type as the key of the outer dict, and then the actual filter data as the value, e.g.,
+    {
+        "equals": {
+            "property": [...],
+            "value": ...
+        }
+    }
+    We could have modeled it that way with Pydantic, and had an union of pydantic models of all possible filter types.
+    However, validating union types in Pydantic without a discriminator leads to poor error messages. If the filter
+    data does not comply with any of the union types, Pydantic will give one error message per union type. For exampl,
+    if the user writes
+    {
+        "equals": {
+            "property": "my_property"  # Should be a list,
+            "value": 'my_value'
+            }
+    }
+    Pydantic will give 15 error messages, one for each filter type in the union, saying that the data does not
+    comply with that filter type. This is not very user-friendly.
+
+    Instead, we introduce an internal field "filter_type" inside the filter data models, and use that as a
+    discriminator. This will enable the validation to be two steps. First, we validate the outer key and
+    that it is a known filter type. Then, we move that key inside the filter data as the "filter_type" field, and
+    validate the filter data against the correct model based on that discriminator.
+
+
+    This function transforms the data from the outer-key format to the inner-key format. For example, it transforms
+    the equals filter form above into
+
+    {
+        "equals": {
+            "property": [...],
+            "value": ...,
+            "filterType": "equals"
+        }
+    }
+    """
+    # legacy filter which we want to ignore
+    if _is_legacy_filter(value):
+        return None
+    if not isinstance(value, dict):
+        return value
+    if len(value) != 1:
+        raise ValueError("Filter data must have exactly one key.")
+    if "filterType" in value:
+        # Already in the correct format
+        return value
+    key, data = next(iter(value.items()))
+    # Check if inner data already has filterType (already processed by a previous recursive call)
+    if isinstance(data, dict) and "filterType" in data:
+        return value
+    if key not in AVAILABLE_FILTERS:
+        raise ValueError(
+            f"Unknown filter type: {key!r}. Available filter types: {humanize_collection(AVAILABLE_FILTERS)}."
+        )
+    if isinstance(data, dict) and key == "not":
+        # Not is a recursive filter, so we need to move the filter key inside its data as well
+        output = _move_filter_key(data.copy())
+        return {key: {"filterType": key, "data": output}}
+    elif isinstance(data, dict):
+        output = data.copy()
+        output["filterType"] = key
+        return {key: output}
+    elif isinstance(data, list) and key in {"and", "or"}:
+        # And and Or are recursive filters, so we need to move the filter key inside each of their data items as well
+        return {key: {"filterType": key, "data": [_move_filter_key(item) for item in data]}}
+    elif isinstance(data, list):
+        # Leaf list filters, hasData and instanceReferences
+        return {key: {"filterType": key, "data": data}}
+    else:
+        # Let the regular validation handle it (possible not an issue)
+        return value
+
+
+def _is_legacy_filter(filter_obj: Any) -> bool:
+    """Check if filter is a legacy filter no longer supported by DMS API"""
+
+    def traverse(obj: Any) -> bool:
+        if isinstance(obj, dict):
+            for key, value in obj.items():
+                if key in LEGACY_FILTERS:
+                    return True
+                if traverse(value):
+                    return True
+        elif isinstance(obj, list):
+            for item in obj:
+                if traverse(item):
+                    return True
+        return False
+
+    return traverse(filter_obj)
+
+
+Filter = Annotated[dict[FilterTypes, FilterData] | None, BeforeValidator(_move_filter_key)]
+
+FilterAdapter: TypeAdapter[Filter] = TypeAdapter(Filter)