cognite-neat 0.127.18__py3-none-any.whl → 0.127.20__py3-none-any.whl

cognite/neat/_data_model/exporters/_table_exporter/writer.py

@@ -5,6 +5,8 @@ from typing import Any, Literal
 
 from cognite.neat._data_model._constants import DEFAULT_MAX_LIST_SIZE, DEFAULT_MAX_LIST_SIZE_DIRECT_RELATIONS
 from cognite.neat._data_model.importers._table_importer.data_classes import (
+    CREATOR_KEY,
+    CREATOR_MARKER,
     DMSContainer,
     DMSEnum,
     DMSNode,
@@ -78,14 +80,34 @@ class DMSTableWriter:
         )
 
     ### Metadata Sheet ###
-    @staticmethod
-    def write_metadata(data_model: DataModelRequest) -> list[MetadataValue]:
-        return [
+    @classmethod
+    def write_metadata(cls, data_model: DataModelRequest) -> list[MetadataValue]:
+        metadata = [
             MetadataValue(key=key, value=value)
             for key, value in data_model.model_dump(
-                mode="json", by_alias=True, exclude_none=True, exclude={"views"}
+                mode="json", by_alias=True, exclude_none=True, exclude={"views", "description"}
             ).items()
         ]
+        if data_model.description:
+            description, creator = cls._serialize_description(data_model.description)
+            if description:
+                metadata.append(MetadataValue(key="description", value=description))
+            if creator:
+                metadata.append(MetadataValue(key=CREATOR_KEY, value=creator))
+        return metadata
+
+    @staticmethod
+    def _serialize_description(description: str | None) -> tuple[str | None, str | None]:
+        """DataModelRequest does not have a 'creator' field, this is a special addition that the Neat tables
+        format supports (and recommends using). If the data model was created using Neat, the suffix of the
+        description will be Creator: <creator>. This function extracts that information."""
+        if description is None:
+            return None, None
+        if CREATOR_MARKER not in description:
+            return description, None
+
+        description, creator = description.rsplit(CREATOR_MARKER, 1)
+        return description.rstrip(), creator.strip()
 
     ### Container Properties Sheet ###
 

cognite/neat/_data_model/importers/_table_importer/data_classes.py

@@ -24,6 +24,10 @@ from cognite.neat.v0.core._data_model.models.entities import (
     RawFilter,
 )
 
+# This marker is used to identify creator in the description field.
+CREATOR_MARKER = "Creator: "
+CREATOR_KEY = "creator"
+
 
 def parse_entity_str(v: str) -> ParsedEntity:
     if isinstance(v, ParsedEntity):

cognite/neat/_data_model/importers/_table_importer/reader.py

@@ -9,6 +9,7 @@ from cognite.neat._data_model.models.dms import (
     Constraint,
     ConstraintAdapter,
     ContainerPropertyDefinition,
+    ContainerReference,
     ContainerRequest,
     DataModelRequest,
     Index,
@@ -17,17 +18,19 @@ from cognite.neat._data_model.models.dms import (
     RequestSchema,
     SpaceRequest,
     UniquenessConstraintDefinition,
+    ViewReference,
     ViewRequest,
     ViewRequestProperty,
     ViewRequestPropertyAdapter,
 )
+from cognite.neat._data_model.models.dms._constants import DATA_MODEL_DESCRIPTION_MAX_LENGTH
 from cognite.neat._data_model.models.entities import ParsedEntity, parse_entity
 from cognite.neat._exceptions import DataModelImportException
 from cognite.neat._issues import ModelSyntaxError
 from cognite.neat._utils.text import humanize_collection
 from cognite.neat._utils.validation import ValidationContext, humanize_validation_error
 
-from .data_classes import DMSContainer, DMSEnum, DMSNode, DMSProperty, DMSView, TableDMS
+from .data_classes import CREATOR_KEY, CREATOR_MARKER, DMSContainer, DMSEnum, DMSNode, DMSProperty, DMSView, TableDMS
 from .source import TableSource
 
 T_BaseModel = TypeVar("T_BaseModel", bound=BaseModel)
@@ -169,7 +172,9 @@ class DMSTableReader:
         space_request = self.read_space(self.default_space)
         node_types = self.read_nodes(tables.nodes)
         enum_collections = self.read_enum_collections(tables.enum)
-        read = self.read_properties(tables.properties, enum_collections)
+        container_ref_by_entity = self.read_entity_by_container_ref(tables.containers)
+        view_ref_by_entity = self.read_entity_by_view_ref(tables.views)
+        read = self.read_properties(tables.properties, enum_collections, container_ref_by_entity, view_ref_by_entity)
         processed = self.process_properties(read)
         containers = self.read_containers(tables.containers, processed)
         views, valid_view_entities = self.read_views(tables.views, processed.view)
@@ -207,18 +212,50 @@ class DMSTableReader:
             }
         return enum_collections
 
+    def read_entity_by_container_ref(self, containers: list[DMSContainer]) -> dict[ContainerReference, ParsedEntity]:
+        entity_by_container_ref: dict[ContainerReference, ParsedEntity] = {}
+        for container in containers:
+            data = self._create_container_ref(container.container)
+            try:
+                container_ref = ContainerReference.model_validate(data)
+            except ValidationError:
+                # Error will be reported when reading the containers
+                continue
+            entity_by_container_ref[container_ref] = container.container
+        return entity_by_container_ref
+
+    def read_entity_by_view_ref(self, views: list[DMSView]) -> dict[ViewReference, ParsedEntity]:
+        entity_by_view_ref: dict[ViewReference, ParsedEntity] = {}
+        for view in views:
+            data = self._create_view_ref(view.view)
+            try:
+                view_ref = ViewReference.model_validate(data)
+            except ValidationError:
+                # Error will be reported when reading the views
+                continue
+            entity_by_view_ref[view_ref] = view.view
+        return entity_by_view_ref
+
     def read_properties(
-        self, properties: list[DMSProperty], enum_collections: dict[str, dict[str, Any]]
+        self,
+        properties: list[DMSProperty],
+        enum_collections: dict[str, dict[str, Any]],
+        container_ref_by_entity: dict[ContainerReference, ParsedEntity],
+        view_ref_by_entity: dict[ViewReference, ParsedEntity],
     ) -> ReadProperties:
         read = ReadProperties()
+        view_entities = set(view_ref_by_entity.values())
+        container_entities = set(container_ref_by_entity.values())
         for row_no, prop in enumerate(properties):
-            self._process_view_property(prop, read, row_no)
+            self._process_view_property(prop, read, row_no, view_ref_by_entity, view_entities)
             if prop.container is None or prop.container_property is None:
                 # This is when the property is an edge or reverse direct relation property.
                 continue
-            self._process_container_property(prop, read, enum_collections, row_no)
-            self._process_index(prop, read, row_no)
-            self._process_constraint(prop, read, row_no)
+            self._process_container_property(
+                prop, read, enum_collections, row_no, container_ref_by_entity, container_entities
+            )
+            self._process_index(prop, read, row_no, container_ref_by_entity, container_entities)
+            self._process_constraint(prop, read, row_no, container_ref_by_entity, container_entities)
         return read
 
     def process_properties(self, read: ReadProperties) -> ProcessedProperties:
@@ -370,31 +407,91 @@ class DMSTableReader:
             constraints[container_entity][constraint_id] = constraint
         return constraints
 
-    def _process_view_property(self, prop: DMSProperty, read: ReadProperties, row_no: int) -> None:
+    def _process_view_property(
+        self,
+        prop: DMSProperty,
+        read: ReadProperties,
+        row_no: int,
+        view_ref_by_entity: dict[ViewReference, ParsedEntity],
+        view_entities: set[ParsedEntity],
+    ) -> None:
         loc = (self.Sheets.properties, row_no)
         data = self.read_view_property(prop, loc)
         view_prop = self._validate_adapter(ViewRequestPropertyAdapter, data, loc)
-        if view_prop is not None:
-            read.view[(prop.view, prop.view_property)].append(
-                # MyPy has a very strange complaint here. It complains that given type is not expected type,
-                # even though they are exactly the same.
-                ReadViewProperty(prop.container_property, row_no, view_prop)  # type: ignore[arg-type]
+        if view_prop is None:
+            return None
+        if prop.view in view_entities:
+            read.view[(prop.view, prop.view_property)].append(ReadViewProperty(prop.view_property, row_no, view_prop))
+            return None
+        # The view entity was not found in the views table. This could either be because the view is missing,
+        # or because either the view entity in the Properties table and the View table are specified with/without
+        # default space/version inconsistently.
+        try:
+            view_ref = ViewReference.model_validate(self._create_view_ref(prop.view))
+        except ValidationError:
+            # Error will be reported when reading the views
+            return None
+        if view_ref in view_ref_by_entity:
+            view_ref_entity = view_ref_by_entity[view_ref]
+            read.view[(view_ref_entity, prop.view_property)].append(
+                ReadViewProperty(prop.view_property, row_no, view_prop)
+            )
+        else:
+            self.errors.append(
+                ModelSyntaxError(
+                    message=(
+                        f"In {self.source.location(loc)} the View '{prop.view!s}' "
+                        f"was not found in the {self.Sheets.views!r} table."
+                    )
+                )
             )
         return None
 
     def _process_container_property(
-        self, prop: DMSProperty, read: ReadProperties, enum_collections: dict[str, dict[str, Any]], row_no: int
+        self,
+        prop: DMSProperty,
+        read: ReadProperties,
+        enum_collections: dict[str, dict[str, Any]],
+        row_no: int,
+        container_ref_by_entity: dict[ContainerReference, ParsedEntity],
+        container_entities: set[ParsedEntity],
     ) -> None:
         loc = (self.Sheets.properties, row_no)
         data = self.read_container_property(prop, enum_collections, loc=loc)
         container_prop = self._validate_obj(ContainerPropertyDefinition, data, loc)
-        if container_prop is not None and prop.container and prop.container_property:
+        if container_prop is None:
+            return None
+        if not (prop.container and prop.container_property):
+            return None
+        if prop.container in container_entities:
             read.container[(prop.container, prop.container_property)].append(
                 ReadContainerProperty(prop.container_property, row_no, container_prop)
             )
+            return None
+        # The container entity was not found in the containers table. This could either be because the container
+        # is missing, or because either the container entity in the Properties table and the Container table are
+        # specified with/without default space/version inconsistently.
+        try:
+            container_ref = ContainerReference.model_validate(self._create_container_ref(prop.container))
+        except ValidationError:
+            # Error will be reported when reading the containers
+            return None
+        if container_ref in container_ref_by_entity:
+            container_ref_entity = container_ref_by_entity[container_ref]
+            read.container[(container_ref_entity, prop.container_property)].append(
+                ReadContainerProperty(prop.container_property, row_no, container_prop)
+            )
+        # Container can be in CDF, this will be reported by a validator later.
         return None
 
-    def _process_index(self, prop: DMSProperty, read: ReadProperties, row_no: int) -> None:
+    def _process_index(
+        self,
+        prop: DMSProperty,
+        read: ReadProperties,
+        row_no: int,
+        container_ref_by_entity: dict[ContainerReference, ParsedEntity],
+        container_entities: set[ParsedEntity],
+    ) -> None:
         if prop.index is None or prop.container_property is None or prop.container is None:
             return
 
@@ -405,11 +502,41 @@ class DMSTableReader:
             if created is None:
                 continue
             order = self._read_order(index.properties, loc)
-            read.indices[(prop.container, index.suffix)].append(
-                ReadIndex(
-                    prop_id=prop.container_property, order=order, row_no=row_no, index_id=index.suffix, index=created
+
+            if prop.container in container_entities:
+                read.indices[(prop.container, index.suffix)].append(
+                    ReadIndex(
+                        prop_id=prop.container_property,
+                        order=order,
+                        row_no=row_no,
+                        index_id=index.suffix,
+                        index=created,
+                    )
                 )
-            )
+                continue
+            # The container entity was not found in the containers table. This could either be because the
+            # container is missing, or because either the container entity in the Properties table and the
+            # Container table are specified with/without default space/version inconsistently.
+
+            try:
+                container_ref = ContainerReference.model_validate(self._create_container_ref(prop.container))
+            except ValidationError:
+                # Error will be reported when reading the containers
+                continue
+            if container_ref in container_ref_by_entity:
+                container_ref_entity = container_ref_by_entity[container_ref]
+                read.indices[(container_ref_entity, index.suffix)].append(
+                    ReadIndex(
+                        prop_id=prop.container_property,
+                        order=order,
+                        row_no=row_no,
+                        index_id=index.suffix,
+                        index=created,
+                    )
+                )
+            else:
+                # Error is reported when reading the property.
+                ...
 
     def _read_order(self, properties: dict[str, Any], loc: tuple[str | int, ...]) -> int | None:
         if "order" not in properties:
@@ -433,7 +560,14 @@ class DMSTableReader:
             **index.properties,
         }
 
-    def _process_constraint(self, prop: DMSProperty, read: ReadProperties, row_no: int) -> None:
+    def _process_constraint(
+        self,
+        prop: DMSProperty,
+        read: ReadProperties,
+        row_no: int,
+        container_ref_by_entity: dict[ContainerReference, ParsedEntity],
+        container_entities: set[ParsedEntity],
+    ) -> None:
         if prop.constraint is None or prop.container_property is None or prop.container is None:
             return
         loc = (self.Sheets.properties, row_no, self.PropertyColumns.constraint)
@@ -443,15 +577,40 @@ class DMSTableReader:
             if created is None:
                 continue
             order = self._read_order(constraint.properties, loc)
-            read.constraints[(prop.container, constraint.suffix)].append(
-                ReadConstraint(
-                    prop_id=prop.container_property,
-                    order=order,
-                    constraint_id=constraint.suffix,
-                    row_no=row_no,
-                    constraint=created,
+
+            if prop.container in container_entities:
+                read.constraints[(prop.container, constraint.suffix)].append(
+                    ReadConstraint(
+                        prop_id=prop.container_property,
+                        order=order,
+                        constraint_id=constraint.suffix,
+                        row_no=row_no,
+                        constraint=created,
+                    )
                 )
-            )
+                continue
+            # The container entity was not found in the containers table. This could either be because the
+            # container is missing, or because either the container entity in the Properties table and the
+            # Container table are specified with/without default space/version inconsistently.
+            try:
+                container_ref = ContainerReference.model_validate(self._create_container_ref(prop.container))
+            except ValidationError:
+                # Error will be reported when reading the containers
+                continue
+            if container_ref in container_ref_by_entity:
+                container_ref_entity = container_ref_by_entity[container_ref]
+                read.constraints[(container_ref_entity, constraint.suffix)].append(
+                    ReadConstraint(
+                        prop_id=prop.container_property,
+                        order=order,
+                        constraint_id=constraint.suffix,
+                        row_no=row_no,
+                        constraint=created,
+                    )
+                )
+            else:
+                # Error is reported when reading the property.
+                ...
 
     @staticmethod
     def read_property_constraint(constraint: ParsedEntity, prop_id: str) -> dict[str, Any]:
@@ -733,16 +892,45 @@ class DMSTableReader:
         return views_requests, set(rows_by_seen.keys())
 
     def read_data_model(self, tables: TableDMS, valid_view_entities: set[ParsedEntity]) -> DataModelRequest:
-        data = {
+        data: dict[str, Any] = {
             **{meta.key: meta.value for meta in tables.metadata},
             "views": [self._create_view_ref(view.view) for view in tables.views if view.view in valid_view_entities],
         }
+        if description := self._create_description_field(data):
+            data["description"] = description
         model = self._validate_obj(DataModelRequest, data, (self.Sheets.metadata,), field_name="value")
         if model is None:
             # This is the last step, so we can raise the error here.
             raise DataModelImportException(self.errors) from None
         return model
 
+    def _create_description_field(self, data: dict[str, Any]) -> str | None:
+        """DataModelRequest does not have a 'creator' field, this is a special addition that the Neat tables
+        format supports (and recommends using). To keep it, Neat adds it to the suffix of the description field.
+        """
+        if CREATOR_KEY not in data and CREATOR_KEY.title() not in data:
+            return None
+        creator_val = data.pop(CREATOR_KEY, data.pop(CREATOR_KEY.title(), None))
+
+        if not creator_val:
+            return None
+
+        creator = str(creator_val)
+        # We do a split/join to clean up any spaces around commas. Ensuring that we have a consistent
+        # canonical format.
+        cleaned_creator = ", ".join(item.strip() for item in creator.split(","))
+        if not cleaned_creator:
+            return None
+        suffix = f"{CREATOR_MARKER}{cleaned_creator}"
+        description = data.get("description", "")
+        if len(description) + len(suffix) > DATA_MODEL_DESCRIPTION_MAX_LENGTH:
+            description = description[: DATA_MODEL_DESCRIPTION_MAX_LENGTH - len(suffix) - 4] + "..."
+        if description:
+            description = f"{description} {suffix}"
+        else:
+            description = suffix
+        return description
+
     def _parse_entity(self, entity: str, loc: tuple[str | int, ...]) -> ParsedEntity | None:
         try:
             parsed = parse_entity(entity)

cognite/neat/_data_model/models/dms/__init__.py

@@ -58,6 +58,27 @@ from ._references import (
     ViewReference,
 )
 from ._schema import RequestSchema
+from ._view_filter import (
+    AVAILABLE_FILTERS,
+    AndFilter,
+    ContainsAllFilterData,
+    ContainsAnyFilterData,
+    EqualsFilterData,
+    ExistsFilterData,
+    Filter,
+    FilterAdapter,
+    FilterDataDefinition,
+    HasDataFilter,
+    InFilterData,
+    InstanceReferencesFilterData,
+    MatchAllFilterData,
+    NestedFilterData,
+    NotFilter,
+    OrFilter,
+    OverlapsFilterData,
+    PrefixFilterData,
+    RangeFilterData,
+)
 from ._view_property import (
     ConnectionPropertyDefinition,
     ConstraintOrIndexState,
@@ -81,8 +102,10 @@ from ._views import (
 )
 
 __all__ = [
+    "AVAILABLE_FILTERS",
     "DMS_DATA_TYPES",
     "APIResource",
+    "AndFilter",
     "BaseModelObject",
     "BooleanProperty",
     "BtreeIndex",
@@ -99,6 +122,8 @@ __all__ = [
     "ContainerReference",
     "ContainerRequest",
     "ContainerResponse",
+    "ContainsAllFilterData",
+    "ContainsAnyFilterData",
     "DataModelBody",
     "DataModelReference",
     "DataModelRequest",
@@ -110,23 +135,38 @@ __all__ = [
     "DirectNodeRelation",
     "EnumProperty",
     "EnumValue",
+    "EqualsFilterData",
+    "ExistsFilterData",
     "FileCDFExternalIdReference",
+    "Filter",
+    "FilterAdapter",
+    "FilterDataDefinition",
     "Float32Property",
     "Float64Property",
     "FloatProperty",
+    "HasDataFilter",
+    "InFilterData",
     "Index",
     "IndexAdapter",
     "IndexDefinition",
+    "InstanceReferencesFilterData",
     "Int32Property",
     "Int64Property",
     "InvertedIndex",
     "JSONProperty",
     "ListablePropertyTypeDefinition",
+    "MatchAllFilterData",
     "MultiEdgeProperty",
     "MultiReverseDirectRelationPropertyRequest",
     "MultiReverseDirectRelationPropertyResponse",
+    "NestedFilterData",
     "NodeReference",
+    "NotFilter",
+    "OrFilter",
+    "OverlapsFilterData",
+    "PrefixFilterData",
     "PropertyTypeDefinition",
+    "RangeFilterData",
     "RequestSchema",
     "RequiresConstraintDefinition",
     "Resource",

cognite/neat/_data_model/models/dms/_constants.py

@@ -4,6 +4,7 @@ CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN = r"^[a-zA-Z0-9][a-zA-Z0-9_-]{0
 INSTANCE_ID_PATTERN = r"^[^\x00]{1,256}$"
 ENUM_VALUE_IDENTIFIER_PATTERN = r"^[_A-Za-z][_0-9A-Za-z]{0,127}$"
 DM_VERSION_PATTERN = r"^[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])?$"
+DATA_MODEL_DESCRIPTION_MAX_LENGTH = 1024
 FORBIDDEN_ENUM_VALUES = frozenset({"true", "false", "null"})
 FORBIDDEN_SPACES = frozenset(["space", "cdf", "dms", "pg3", "shared", "system", "node", "edge"])
 FORBIDDEN_CONTAINER_AND_VIEW_EXTERNAL_IDS = frozenset(

cognite/neat/_data_model/models/dms/_data_model.py

@@ -6,6 +6,7 @@ from pydantic_core.core_schema import FieldSerializationInfo
 
 from ._base import APIResource, Resource, WriteableResource
 from ._constants import (
+    DATA_MODEL_DESCRIPTION_MAX_LENGTH,
     DM_EXTERNAL_ID_PATTERN,
     DM_VERSION_PATTERN,
     SPACE_FORMAT_PATTERN,
@@ -46,7 +47,7 @@ class DataModel(Resource, APIResource[DataModelReference], ABC):
     description: str | None = Field(
         default=None,
         description="Description of the data model.",
-        max_length=1024,
+        max_length=DATA_MODEL_DESCRIPTION_MAX_LENGTH,
     )
     # The API supports View here, but in Neat we will only use ViewReference
     views: list[ViewReference] | None = Field(

cognite/neat/_data_model/models/dms/_view_filter.py

@@ -0,0 +1,282 @@
+from abc import ABC
+from typing import Annotated, Any, Literal, TypeAlias, get_args
+
+from pydantic import BeforeValidator, Field, JsonValue, TypeAdapter, model_serializer
+from pydantic_core.core_schema import FieldSerializationInfo
+
+from cognite.neat._utils.text import humanize_collection
+from cognite.neat._utils.useful_types import BaseModelObject
+
+from ._references import ContainerReference, NodeReference, ViewReference
+
+# Base classes and helpers
+
+
+class Parameter(BaseModelObject):
+    parameter: str
+
+
+class FilterDataDefinition(BaseModelObject, ABC):
+    """Base class for filter data models."""
+
+    # This is an internal field used for discriminating between filter types. It is not part of the actual
+    # data sent to or received from the API. See the _move_filter_key function for more details.
+    filter_type: str = Field(..., exclude=True)
+
+
+class PropertyReference(FilterDataDefinition, ABC):
+    """Represents the property path in filters."""
+
+    property: list[str] = Field(..., min_length=2, max_length=3)
+
+
+## Leaf filters that follows the standard pattern
+
+
+class EqualsFilterData(PropertyReference):
+    filter_type: Literal["equals"] = Field("equals", exclude=True)
+    value: JsonValue | PropertyReference
+
+
+class InFilterData(PropertyReference):
+    filter_type: Literal["in"] = Field("in", exclude=True)
+    values: list[JsonValue] | PropertyReference
+
+
+class RangeFilterData(PropertyReference):
+    filter_type: Literal["range"] = Field("range", exclude=True)
+    gt: str | int | float | PropertyReference | None = None
+    gte: str | int | float | PropertyReference | None = None
+    lt: str | int | float | PropertyReference | None = None
+    lte: str | int | float | PropertyReference | None = None
+
+
+class PrefixFilterData(PropertyReference):
+    filter_type: Literal["prefix"] = Field("prefix", exclude=True)
+    value: str | Parameter
+
+
+class ExistsFilterData(PropertyReference):
+    filter_type: Literal["exists"] = Field("exists", exclude=True)
+
+
+class ContainsAnyFilterData(PropertyReference):
+    filter_type: Literal["containsAny"] = Field("containsAny", exclude=True)
+    values: list[JsonValue] | PropertyReference
+
+
+class ContainsAllFilterData(PropertyReference):
+    filter_type: Literal["containsAll"] = Field("containsAll", exclude=True)
+    values: list[JsonValue] | PropertyReference
+
+
+class MatchAllFilterData(FilterDataDefinition):
+    filter_type: Literal["matchAll"] = Field("matchAll", exclude=True)
+
+
+class NestedFilterData(PropertyReference):
+    filter_type: Literal["nested"] = Field("nested", exclude=True)
+    scope: list[str] = Field(..., min_length=1, max_length=3)
+    filter: "Filter"
+
+
+class OverlapsFilterData(PropertyReference):
+    filter_type: Literal["overlaps"] = Field("overlaps", exclude=True)
+    start_property: list[str] = Field(..., min_length=1, max_length=3)
+    end_property: list[str] = Field(..., min_length=1, max_length=3)
+    gt: str | int | float | PropertyReference | None = None
+    gte: str | int | float | PropertyReference | None = None
+    lt: str | int | float | PropertyReference | None = None
+    lte: str | int | float | PropertyReference | None = None
+
+
+class ListFilterDataDefinition(FilterDataDefinition, ABC):
+    """Base class for filters that operate on lists of values."""
+
+    data: list[Any]
+
+
+## Leaf filters with custom serialization logic due to creativity in the API design
+
+
+class HasDataFilter(ListFilterDataDefinition):
+    filter_type: Literal["hasData"] = Field("hasData", exclude=True)
+    data: list[ViewReference | ContainerReference]
+
+    # MyPy complains about thet signature of the method here, even though its compatible with the pydantic source code.
+    # And tests are passing fine.
+    @model_serializer(mode="plain")  # type: ignore[type-var]
+    def serialize_model(self, info: FieldSerializationInfo) -> list[dict[str, Any]]:
+        output: list[dict[str, Any]] = []
+        for item in self.data:
+            item_dict = item.model_dump(**vars(info))
+            if isinstance(item, ViewReference):
+                item_dict["type"] = "view"
+            elif isinstance(item, ContainerReference):
+                item_dict["type"] = "container"
+            output.append(item_dict)
+        return output
+
+
+class InstanceReferencesFilterData(ListFilterDataDefinition):
+    filter_type: Literal["instanceReferences"] = Field("instanceReferences", exclude=True)
+    data: list[NodeReference]
+
+    # MyPy complains about thet signature of the method here, even though its compatible with the pydantic source code.
+    # And tests are passing fine.
+    @model_serializer(mode="plain")  # type: ignore[type-var]
+    def serialize_model(self, info: FieldSerializationInfo) -> list[dict[str, Any]]:
+        return [item.model_dump(**vars(info)) for item in self.data]
+
+
+## Logical filters combining other filters
+
+
+class AndFilter(ListFilterDataDefinition):
+    filter_type: Literal["and"] = Field("and", exclude=True)
+    data: "list[Filter]"
+
+    # MyPy complains about thet signature of the method here, even though its compatible with the pydantic source code.
+    # And tests are passing fine.
+    @model_serializer(mode="plain")  # type: ignore[type-var]
+    def serialize_model(self, info: FieldSerializationInfo) -> list[dict[str, Any]]:
+        return [FilterAdapter.dump_python(item, **vars(info)) for item in self.data]
+
+
+class OrFilter(ListFilterDataDefinition):
+    filter_type: Literal["or"] = Field("or", exclude=True)
+    data: "list[Filter]"
+
+    # MyPy complains about thet signature of the method here, even though its compatible with the pydantic source code.
+    # And tests are passing fine.
+    @model_serializer(mode="plain")  # type: ignore[type-var]
+    def serialize_model(self, info: FieldSerializationInfo) -> list[dict[str, Any]]:
+        return [FilterAdapter.dump_python(item, **vars(info)) for item in self.data]
+
+
+class NotFilter(FilterDataDefinition):
+    filter_type: Literal["not"] = Field("not", exclude=True)
+    data: "Filter"
+
+    # MyPy complains about thet signature of the method here, even though its compatible with the pydantic source code.
+    # And tests are passing fine.
+    @model_serializer(mode="plain")  # type: ignore[type-var]
+    def serialize_model(self, info: FieldSerializationInfo) -> dict[str, Any]:
+        return FilterAdapter.dump_python(self.data, **vars(info))
+
+
+FilterData = Annotated[
+    EqualsFilterData
+    | PrefixFilterData
+    | InFilterData
+    | RangeFilterData
+    | ExistsFilterData
+    | ContainsAnyFilterData
+    | ContainsAllFilterData
+    | MatchAllFilterData
+    | NestedFilterData
+    | OverlapsFilterData
+    | HasDataFilter
+    | InstanceReferencesFilterData
+    | AndFilter
+    | OrFilter
+    | NotFilter,
+    Field(discriminator="filter_type"),
+]
+
+
+FilterTypes: TypeAlias = Literal[
+    "equals",
+    "prefix",
+    "in",
+    "range",
+    "exists",
+    "containsAny",
+    "containsAll",
+    "matchAll",
+    "nested",
+    "overlaps",
+    "hasData",
+    "instanceReferences",
+    "and",
+    "or",
+    "not",
+]
+
+AVAILABLE_FILTERS: frozenset[str] = frozenset(get_args(FilterTypes))
+
+
+def _move_filter_key(value: Any) -> Any:
+    """The DMS API filters have an unusual structure.
+
+    It has the filter type as the key of the outer dict, and then the actual filter data as the value, e.g.,
+    {
+        "equals": {
+            "property": [...],
+            "value": ...
+        }
+    }
+    We could have modeled it that way with Pydantic, and had an union of pydantic models of all possible filter types.
+    However, validating union types in Pydantic without a discriminator leads to poor error messages. If the filter
+    data does not comply with any of the union types, Pydantic will give one error message per union type. For exampl,
+    if the user writes
+    {
+        "equals": {
+            "property": "my_property"  # Should be a list,
+            "value": 'my_value'
+            }
+    }
+    Pydantic will give 15 error messages, one for each filter type in the union, saying that the data does not
+    comply with that filter type. This is not very user-friendly.
+
+    Instead, we introduce an internal field "filter_type" inside the filter data models, and use that as a
+    discriminator. This will enable the validation to be two steps. First, we validate the outer key and
+    that it is a known filter type. Then, we move that key inside the filter data as the "filter_type" field, and
+    validate the filter data against the correct model based on that discriminator.
+
+
+    This function transforms the data from the outer-key format to the inner-key format. For example, it transforms
+    the equals filter form above into
+
+    {
+        "equals": {
+            "property": [...],
+            "value": ...,
+            "filterType": "equals"
+        }
+    }
+    """
+    if not isinstance(value, dict):
+        return value
+    if len(value) != 1:
+        raise ValueError("Filter data must have exactly one key.")
+    if "filterType" in value:
+        # Already in the correct format
+        return value
+    key, data = next(iter(value.items()))
+    if key not in AVAILABLE_FILTERS:
+        raise ValueError(
+            f"Unknown filter type: {key!r}. Available filter types: {humanize_collection(AVAILABLE_FILTERS)}."
+        )
+    if isinstance(data, dict) and key == "not":
+        # Not is a recursive filter, so we need to move the filter key inside its data as well
+        output = _move_filter_key(data.copy())
+        return {key: {"filterType": key, "data": output}}
+    elif isinstance(data, dict):
+        output = data.copy()
+        output["filterType"] = key
+        return {key: output}
+    elif isinstance(data, list) and key in {"and", "or"}:
+        # And and Or are recursive filters, so we need to move the filter key inside each of their data items as well
+        return {key: {"filterType": key, "data": [_move_filter_key(item) for item in data]}}
+    elif isinstance(data, list):
+        # Leaf list filters, hasData and instanceReferences
+        return {key: {"filterType": key, "data": data}}
+    else:
+        # Let the regular validation handle it (possible not an issue)
+        return value
+
+
+Filter = Annotated[dict[FilterTypes, FilterData], BeforeValidator(_move_filter_key)]
+
+FilterAdapter: TypeAdapter[Filter] = TypeAdapter(Filter)

cognite/neat/_data_model/models/dms/_views.py

@@ -2,7 +2,7 @@ import re
 from abc import ABC
 from typing import Any, Literal, TypeVar
 
-from pydantic import Field, JsonValue, field_serializer, field_validator, model_validator
+from pydantic import Field, field_serializer, field_validator, model_validator
 from pydantic_core.core_schema import FieldSerializationInfo
 
 from cognite.neat._utils.text import humanize_collection
@@ -18,6 +18,7 @@ from ._constants import (
     SPACE_FORMAT_PATTERN,
 )
 from ._references import ContainerReference, NodeReference, ViewReference
+from ._view_filter import Filter
 from ._view_property import (
     EdgeProperty,
     ViewCorePropertyResponse,
@@ -56,9 +57,9 @@ class View(Resource, APIResource[ViewReference], ABC):
         description="Description of the view.",
         max_length=1024,
     )
-    filter: dict[str, JsonValue] | None = Field(
+    filter: Filter | None = Field(
         default=None,
-        description="A filter Domain Specific Language (DSL) used to create advanced filter queries.",
+        description="A filter Domain Specific Language (DSL) used to select which instances the view should include.",
     )
     implements: list[ViewReference] | None = Field(
         default=None,

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.127.18"
+__version__ = "0.127.20"
 __engine__ = "^2.0.4"

{cognite_neat-0.127.18.dist-info → cognite_neat-0.127.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite-neat
-Version: 0.127.18
+Version: 0.127.20
 Summary: Knowledge graph transformation
 Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
 Project-URL: Homepage, https://cognite-neat.readthedocs-hosted.com/

{cognite_neat-0.127.18.dist-info → cognite_neat-0.127.20.dist-info}/RECORD

@@ -1,7 +1,7 @@
 cognite/neat/__init__.py,sha256=Lo4DbjDOwnhCYUoAgPp5RG1fDdF7OlnomalTe7n1ydw,211
 cognite/neat/_exceptions.py,sha256=ox-5hXpee4UJlPE7HpuEHV2C96aLbLKo-BhPDoOAzhA,1650
 cognite/neat/_issues.py,sha256=wH1mnkrpBsHUkQMGUHFLUIQWQlfJ_qMfdF7q0d9wNhY,1871
-cognite/neat/_version.py,sha256=RpJs0sO0_LsEewFOvQbwxwfMhPN1BfsPoPZai6GswuA,47
+cognite/neat/_version.py,sha256=x1it8CkOgLQbQkRRhDB46FfLXgM0aI8us2ZRTHFumD4,47
 cognite/neat/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/v1.py,sha256=owqW5Mml2DSZx1AvPvwNRTBngfhBNrQ6EH-7CKL7Jp0,61
 cognite/neat/_client/__init__.py,sha256=75Bh7eGhaN4sOt3ZcRzHl7pXaheu1z27kmTHeaI05vo,114
@@ -33,14 +33,14 @@ cognite/neat/_data_model/exporters/_base.py,sha256=rG_qAU5i5Hh5hUMep2UmDFFZID4x3
 cognite/neat/_data_model/exporters/_table_exporter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/_data_model/exporters/_table_exporter/exporter.py,sha256=4BPu_Chtjh1EyOaKbThXYohsqllVOkCbSoNekNZuBXc,5159
 cognite/neat/_data_model/exporters/_table_exporter/workbook.py,sha256=1Afk1WqeNe9tiNeSAm0HrF8jTQ1kTbIv1D9hMztKwO8,18482
-cognite/neat/_data_model/exporters/_table_exporter/writer.py,sha256=X5q0kGQA4LuIbT0DLpPbKBjut1x9JELBnc9PTFP8O4o,18163
+cognite/neat/_data_model/exporters/_table_exporter/writer.py,sha256=QsO2BWB-_Jw_hpawHtG26NOnLu6wwtDosT-c1acNLPw,19270
 cognite/neat/_data_model/importers/__init__.py,sha256=dHnKnC_AXk42z6wzEHK15dxIOh8xSEkuUf_AFRZls0E,193
 cognite/neat/_data_model/importers/_api_importer.py,sha256=H8Ow3Tt7utuAuBhC6s7yWvhGqunHAtE0r0XRsVAr6IE,7280
 cognite/neat/_data_model/importers/_base.py,sha256=NRB0FcEBj4GaethU68nRffBfTedBBA866A3zfJNfmiQ,433
 cognite/neat/_data_model/importers/_table_importer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cognite/neat/_data_model/importers/_table_importer/data_classes.py,sha256=5gFeAjAYFlzOBb9cfKZWeih5psKynpJwPMNel7h6cO8,9496
+cognite/neat/_data_model/importers/_table_importer/data_classes.py,sha256=7oy0dYXj2lW2F-9jzrosSAIlDBzcAewMu8e5sU5lHPw,9618
 cognite/neat/_data_model/importers/_table_importer/importer.py,sha256=lQ4_Gpv0haEwQEDYZJaxtR9dL6Y0ys9jbjFfWxH6s2o,8870
-cognite/neat/_data_model/importers/_table_importer/reader.py,sha256=v7_PWuOtUQwRTTkZRw4z37G2IF9BtrMozmnnaM9avKE,40759
+cognite/neat/_data_model/importers/_table_importer/reader.py,sha256=I9-zHCpJLo7bj4BabAzSgNBDVUAocdhlvBfy95JkWRw,49451
 cognite/neat/_data_model/importers/_table_importer/source.py,sha256=h7u5ur5oetmvBs3wgj7Ody5uPF21QwxeAceoIhJ5qzo,3300
 cognite/neat/_data_model/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/_data_model/models/conceptual/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -49,12 +49,12 @@ cognite/neat/_data_model/models/conceptual/_concept.py,sha256=0Pk4W2TJ_Y0Z7oPHpz
 cognite/neat/_data_model/models/conceptual/_data_model.py,sha256=mSX0z8i29ufcRUhvC_NPeo2xGidlK3B1n89kngY_SqQ,1695
 cognite/neat/_data_model/models/conceptual/_properties.py,sha256=CpF37vJYBTLT4DH4ZOu2U-JyWtkb_27V8fw52qiaE_k,4007
 cognite/neat/_data_model/models/conceptual/_property.py,sha256=blSZQxX52zaILAtjUkldPzPeysz7wnG-UGSNU5tacI8,4138
-cognite/neat/_data_model/models/dms/__init__.py,sha256=q268ScyyIEv2KbcRtnC86ubk8KFie65mVlQbXG-Au0g,4446
+cognite/neat/_data_model/models/dms/__init__.py,sha256=CW5NPMRrMyY4iyZgqYb8eZkRuwbbXUDSVNMWep3zEPI,5326
 cognite/neat/_data_model/models/dms/_base.py,sha256=931ODXnhrBrzf6vkjqu2IaFz8r2gGxuapn85yN_jkgg,889
-cognite/neat/_data_model/models/dms/_constants.py,sha256=_S-62UpDnBteLN2hLNCXBnHuMYSwK-cEBh3JKslv5GA,1343
+cognite/neat/_data_model/models/dms/_constants.py,sha256=TaoE9kmNVEaTl_dDrZQL7YzgP4K13ff0Rc7nr4zbIgg,1384
 cognite/neat/_data_model/models/dms/_constraints.py,sha256=cyGgDlByXAuSMWJg7Oc25fkp33LsA61M927bCzTWlbo,1458
 cognite/neat/_data_model/models/dms/_container.py,sha256=wtQbNUwtpymltT1jav8wD4kIfjaIYnvhhz1KS0ffAbo,6044
-cognite/neat/_data_model/models/dms/_data_model.py,sha256=FyC5rJUGQmF32qBwVL2lEGf_Gw1Z438FfRr1dSNe9-Y,3151
+cognite/neat/_data_model/models/dms/_data_model.py,sha256=tq_JGNN-1JxG46bhBhunZiLedklYbDXFEfINB0x3a3Q,3219
 cognite/neat/_data_model/models/dms/_data_types.py,sha256=FMt_d5aJD-o3s9VQWyyCVlHk7D_p3RlSNXBP1OACPs4,6424
 cognite/neat/_data_model/models/dms/_http.py,sha256=YIRRowqkphFAYkx3foTeLyPMe9fNnmzhUCBDXe0u9Kk,926
 cognite/neat/_data_model/models/dms/_indexes.py,sha256=ZtXe8ABuRcsAwRIZ9FCanS3uwZHpkOAhvDvjSXtx_Fs,900
@@ -63,8 +63,9 @@ cognite/neat/_data_model/models/dms/_references.py,sha256=x2sK_YnEpWtLED4j8dqrqV
 cognite/neat/_data_model/models/dms/_schema.py,sha256=2JFLcm52smzPdtZ69Lf02UbYAD8I_hpRbI7ZAzdxJJs,641
 cognite/neat/_data_model/models/dms/_space.py,sha256=jEOK_WTATaFDIfuUd6dCaDkW2ysKKQEkSUZf8dMyYWM,1903
 cognite/neat/_data_model/models/dms/_types.py,sha256=5-cgC53AG186OZUqkltv7pMjcGNLuH7Etbn8IUcgk1c,447
+cognite/neat/_data_model/models/dms/_view_filter.py,sha256=cfeEOtRz5SGFI0rmPD3jNl-V6_zJxyxgxYjG3Oz8OEU,10301
 cognite/neat/_data_model/models/dms/_view_property.py,sha256=nJBPmw4KzJOdaQmvRfCE3A4FL-E13OsNUEufI64vLKo,9271
-cognite/neat/_data_model/models/dms/_views.py,sha256=ffNQx6atn7P-dhNBYmrfFAFALitFydGWL7Ome1_h5lY,8633
+cognite/neat/_data_model/models/dms/_views.py,sha256=1yxuwnsUM4WKItEY1hmJbMQKW0q3Dn321NmLmKLJeCM,8657
 cognite/neat/_data_model/models/entities/__init__.py,sha256=7dDyES7fYl9LEREal59F038RdEvfGRpUOc6n_MtSgjU,836
 cognite/neat/_data_model/models/entities/_base.py,sha256=PaNrD29iwxuqTpRWbmESMTxRhhKXmRyDF_cLZEC69dg,3927
 cognite/neat/_data_model/models/entities/_constants.py,sha256=EK9Bus8UgFgxK5cVFMTAqWSl6aWkDe7d59hpUmlHlBs,517
@@ -311,7 +312,7 @@ cognite/neat/v0/session/engine/__init__.py,sha256=D3MxUorEs6-NtgoICqtZ8PISQrjrr4
 cognite/neat/v0/session/engine/_import.py,sha256=1QxA2_EK613lXYAHKQbZyw2yjo5P9XuiX4Z6_6-WMNQ,169
 cognite/neat/v0/session/engine/_interface.py,sha256=3W-cYr493c_mW3P5O6MKN1xEQg3cA7NHR_ev3zdF9Vk,533
 cognite/neat/v0/session/engine/_load.py,sha256=u0x7vuQCRoNcPt25KJBJRn8sJabonYK4vtSZpiTdP4k,5201
-cognite_neat-0.127.18.dist-info/METADATA,sha256=nYE2C-FdpTvrb-C3-64xoiRrHUZbXqVlR9Q4Svw4naQ,9150
-cognite_neat-0.127.18.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_neat-0.127.18.dist-info/licenses/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-0.127.18.dist-info/RECORD,,
+cognite_neat-0.127.20.dist-info/METADATA,sha256=947oG_DqzFdW87-nrRmFBMEpnHVuNcFj0VcHLYd9PAk,9150
+cognite_neat-0.127.20.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_neat-0.127.20.dist-info/licenses/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-0.127.20.dist-info/RECORD,,