cognite-neat 0.88.2__py3-none-any.whl → 0.89.0__py3-none-any.whl

cognite/neat/rules/models/dms/_validation.py

@@ -3,13 +3,24 @@ from typing import Any, ClassVar
 
 from cognite.client import data_modeling as dm
 
-from cognite.neat.issues import IssueList, NeatIssueList
-from cognite.neat.rules import issues
-from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
-from cognite.neat.rules.models._constants import DMS_CONTAINER_SIZE_LIMIT
+from cognite.neat.constants import DMS_CONTAINER_PROPERTY_SIZE_LIMIT
+from cognite.neat.issues import IssueList, NeatError, NeatIssue, NeatIssueList
+from cognite.neat.issues.errors import (
+    PropertyDefinitionDuplicatedError,
+    ResourceChangedError,
+    ResourceNotDefinedError,
+)
+from cognite.neat.issues.warnings import (
+    NotSupportedHasDataFilterLimitWarning,
+    NotSupportedViewContainerLimitWarning,
+)
+from cognite.neat.issues.warnings.user_modeling import (
+    NotNeatSupportedFilterWarning,
+    ViewPropertyLimitWarning,
+)
+from cognite.neat.rules.models._base_rules import DataModelType, ExtensionCategory, SchemaCompleteness
 from cognite.neat.rules.models.data_types import DataType
-from cognite.neat.rules.models.entities import ContainerEntity
-from cognite.neat.rules.models.wrapped_entities import RawFilter
+from cognite.neat.rules.models.entities import ContainerEntity, RawFilter
 
 from ._rules import DMSProperty, DMSRules
 from ._schema import DMSSchema
@@ -51,7 +62,7 @@ class DMSPostValidation:
             if prop.container and prop.container_property:
                 container_properties_by_id[(prop.container, prop.container_property)].append((prop_no, prop))
 
-        errors: list[issues.spreadsheet.InconsistentContainerDefinitionError] = []
+        errors: list[NeatError] = []
         for (container, prop_name), properties in container_properties_by_id.items():
             if len(properties) == 1:
                 continue
@@ -63,42 +74,78 @@ class DMSPostValidation:
             is_all_direct = all(prop.connection == "direct" for _, prop in properties)
             if len(value_types) > 1 and not is_all_direct:
                 errors.append(
-                    issues.spreadsheet.MultiValueTypeError(
+                    PropertyDefinitionDuplicatedError[dm.ContainerId](
                         container_id,
+                        "container",
                         prop_name,
-                        row_numbers,
-                        {v.dms._type if isinstance(v, DataType) else str(v) for v in value_types},
+                        frozenset({v.dms._type if isinstance(v, DataType) else str(v) for v in value_types}),
+                        tuple(row_numbers),
+                        "rows",
                     )
                 )
             list_definitions = {prop.is_list for _, prop in properties if prop.is_list is not None}
             if len(list_definitions) > 1:
                 errors.append(
-                    issues.spreadsheet.MultiValueIsListError(container_id, prop_name, row_numbers, list_definitions)
+                    PropertyDefinitionDuplicatedError[dm.ContainerId](
+                        container_id,
+                        "container",
+                        prop_name,
+                        frozenset(list_definitions),
+                        tuple(row_numbers),
+                        "rows",
+                    )
                 )
             nullable_definitions = {prop.nullable for _, prop in properties if prop.nullable is not None}
             if len(nullable_definitions) > 1:
                 errors.append(
-                    issues.spreadsheet.MultiNullableError(container_id, prop_name, row_numbers, nullable_definitions)
+                    PropertyDefinitionDuplicatedError[dm.ContainerId](
+                        container_id,
+                        "container",
+                        prop_name,
+                        frozenset(nullable_definitions),
+                        tuple(row_numbers),
+                        "rows",
+                    )
                 )
             default_definitions = {prop.default for _, prop in properties if prop.default is not None}
             if len(default_definitions) > 1:
                 errors.append(
-                    issues.spreadsheet.MultiDefaultError(
-                        container_id, prop_name, row_numbers, list(default_definitions)
+                    PropertyDefinitionDuplicatedError[dm.ContainerId](
+                        container_id,
+                        "container",
+                        prop_name,
+                        frozenset(
+                            tuple(f"{k}:{v}" for k, v in def_.items()) if isinstance(def_, dict) else def_
+                            for def_ in default_definitions
+                        ),
+                        tuple(row_numbers),
+                        "rows",
                     )
                 )
             index_definitions = {",".join(prop.index) for _, prop in properties if prop.index is not None}
             if len(index_definitions) > 1:
                 errors.append(
-                    issues.spreadsheet.MultiIndexError(container_id, prop_name, row_numbers, index_definitions)
+                    PropertyDefinitionDuplicatedError[dm.ContainerId](
+                        container_id,
+                        "container",
+                        prop_name,
+                        frozenset(index_definitions),
+                        tuple(row_numbers),
+                        "rows",
+                    )
                 )
             constraint_definitions = {
                 ",".join(prop.constraint) for _, prop in properties if prop.constraint is not None
             }
             if len(constraint_definitions) > 1:
                 errors.append(
-                    issues.spreadsheet.MultiUniqueConstraintError(
-                        container_id, prop_name, row_numbers, constraint_definitions
+                    PropertyDefinitionDuplicatedError[dm.ContainerId](
+                        container_id,
+                        "container",
+                        prop_name,
+                        frozenset(constraint_definitions),
+                        tuple(row_numbers),
+                        "rows",
                     )
                 )
 
@@ -125,32 +172,25 @@ class DMSPostValidation:
             defined_views |= {view.view.as_id() for view in self.rules.last.views}
 
         property_count_by_view: dict[dm.ViewId, int] = defaultdict(int)
-        errors: list[issues.ValidationIssue] = []
+        errors: list[NeatIssue] = []
         for prop_no, prop in enumerate(self.properties):
             view_id = prop.view.as_id()
             if view_id not in defined_views:
                 errors.append(
-                    issues.spreadsheet.NonExistingViewError(
-                        column="View",
-                        row=prop_no,
-                        type="value_error.missing",
-                        view_id=view_id,
-                        msg="",
-                        input=None,
-                        url=None,
-                    )
+                    ResourceNotDefinedError[dm.ViewId](
+                        identifier=view_id,
+                        resource_type="view",
+                        location="Views Sheet",
+                        column_name="View",
+                        row_number=prop_no,
+                        sheet_name="Properties",
+                    ),
                 )
             else:
                 property_count_by_view[view_id] += 1
         for view_id, count in property_count_by_view.items():
-            if count > DMS_CONTAINER_SIZE_LIMIT:
-                errors.append(
-                    issues.dms.ViewSizeWarning(
-                        view_id=view_id,
-                        limit=DMS_CONTAINER_SIZE_LIMIT,
-                        count=count,
-                    )
-                )
+            if count > DMS_CONTAINER_PROPERTY_SIZE_LIMIT:
+                errors.append(ViewPropertyLimitWarning(view_id, count))
         if self.metadata.schema_ is SchemaCompleteness.complete:
             defined_containers = {container.container.as_id() for container in self.containers or []}
             if self.metadata.data_model_type == DataModelType.solution and self.rules.reference:
@@ -161,28 +201,26 @@ class DMSPostValidation:
             for prop_no, prop in enumerate(self.properties):
                 if prop.container and (container_id := prop.container.as_id()) not in defined_containers:
                     errors.append(
-                        issues.spreadsheet.NonExistingContainerError(
-                            column="Container",
-                            row=prop_no,
-                            type="value_error.missing",
-                            container_id=container_id,
-                            msg="",
-                            input=None,
-                            url=None,
+                        ResourceNotDefinedError[dm.ContainerId](
+                            identifier=container_id,
+                            resource_type="container",
+                            location="Containers Sheet",
+                            column_name="Container",
+                            row_number=prop_no,
+                            sheet_name="Properties",
                         )
                     )
             for _container_no, container in enumerate(self.containers or []):
                 for constraint_no, constraint in enumerate(container.constraint or []):
                     if constraint.as_id() not in defined_containers:
                         errors.append(
-                            issues.spreadsheet.NonExistingContainerError(
-                                column="Constraint",
-                                row=constraint_no,
-                                type="value_error.missing",
-                                container_id=constraint.as_id(),
-                                msg="",
-                                input=None,
-                                url=None,
+                            ResourceNotDefinedError[dm.ContainerId](
+                                identifier=constraint.as_id(),
+                                resource_type="container",
+                                location="Containers Sheet",
+                                column_name="Constraint",
+                                row_number=constraint_no,
+                                sheet_name="Properties",
                             )
                         )
         self.issue_list.extend(errors)
@@ -212,10 +250,11 @@ class DMSPostValidation:
                 new_dumped, existing_dumped
             )
             self.issue_list.append(
-                issues.dms.ChangingContainerError(
-                    container_id=container_id,
-                    changed_properties=changed_properties or None,
-                    changed_attributes=changed_attributes or None,
+                ResourceChangedError(
+                    container_id,
+                    "container",
+                    changed_properties=frozenset(changed_properties),
+                    changed_attributes=frozenset(changed_attributes),
                 )
             )
 
@@ -241,10 +280,11 @@ class DMSPostValidation:
                 # Only added new properties, no problem
                 continue
             self.issue_list.append(
-                issues.dms.ChangingViewError(
-                    view_id=view_id,
-                    changed_properties=changed_properties or None,
-                    changed_attributes=changed_attributes or None,
+                ResourceChangedError(
+                    view_id,
+                    "view",
+                    changed_properties=frozenset(changed_properties),
+                    changed_attributes=frozenset(changed_attributes),
                 )
             )
 
@@ -254,9 +294,9 @@ class DMSPostValidation:
 
             if mapped_containers and len(mapped_containers) > 10:
                 self.issue_list.append(
-                    issues.dms.ViewMapsToTooManyContainersWarning(
-                        view_id=view_id,
-                        container_ids=mapped_containers,
+                    NotSupportedViewContainerLimitWarning(
+                        view_id,
+                        len(mapped_containers),
                     )
                 )
                 if (
@@ -265,9 +305,9 @@ class DMSPostValidation:
                     and len(view.filter.dump()["hasData"]) > 10
                 ):
                     self.issue_list.append(
-                        issues.dms.HasDataFilterAppliedToTooManyContainersWarning(
-                            view_id=view_id,
-                            container_ids=mapped_containers,
+                        NotSupportedHasDataFilterLimitWarning(
+                            view_id,
+                            len(view.filter.dump()["hasData"]),
                         )
                     )
 
@@ -275,9 +315,7 @@ class DMSPostValidation:
         for view in self.views:
             if view.filter_ and isinstance(view.filter_, RawFilter):
                 self.issue_list.append(
-                    issues.dms.RawFilterAppliedToViewWarning(
-                        view_id=view.view.as_id(),
-                    )
+                    NotNeatSupportedFilterWarning(view.view.as_id()),
                 )
 
     @staticmethod

cognite/neat/rules/models/domain.py

@@ -1,4 +1,5 @@
 import math
+from dataclasses import dataclass, field
 from typing import Any, ClassVar
 
 from pydantic import Field, field_serializer, field_validator, model_serializer
@@ -7,7 +8,8 @@ from pydantic_core.core_schema import SerializationInfo
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import ClassEntity, ClassEntityList
 
-from ._base import (
+from ._base_input import InputComponent, InputRules
+from ._base_rules import (
     BaseMetadata,
     BaseRules,
     RoleTypes,
@@ -76,3 +78,52 @@ class DomainRules(BaseRules):
                 cls.model_dump(**kwargs) for cls in self.classes or []
             ] or None
         return output
+
+
+@dataclass
+class DomainInputMetadata(InputComponent[DomainMetadata]):
+    creator: str
+
+    @classmethod
+    def _get_verified_cls(cls) -> type[DomainMetadata]:
+        return DomainMetadata
+
+
+@dataclass
+class DomainInputProperty(InputComponent[DomainProperty]):
+    class_: str
+    property_: str
+    value_type: str
+    name: str | None = None
+    description: str | None = None
+    min_count: int | None = None
+    max_count: int | float | None = None
+
+    @classmethod
+    def _get_verified_cls(cls) -> type[DomainProperty]:
+        return DomainProperty
+
+
+@dataclass
+class DomainInputClass(InputComponent[DomainClass]):
+    class_: str
+    name: str | None = None
+    description: str | None = None
+    parent: list[str] | None = None
+
+    @classmethod
+    def _get_verified_cls(cls) -> type[DomainClass]:
+        return DomainClass
+
+
+@dataclass
+class DomainInputRules(InputRules[DomainRules]):
+    metadata: DomainInputMetadata
+    properties: list[DomainInputProperty] = field(default_factory=list)
+    classes: list[DomainInputClass] = field(default_factory=list)
+    last: "DomainInputRules | None" = None
+    reference: "DomainInputRules | None" = None
+
+    @classmethod
+    def _get_verified_cls(cls) -> type[DomainRules]:
+        return DomainRules

cognite/neat/rules/models/entities/__init__.py

@@ -0,0 +1,63 @@
+from ._constants import EntityTypes, Undefined, Unknown
+from ._loaders import load_connection, load_dms_value_type, load_value_type
+from ._multi_value import MultiValueTypeInfo
+from ._single_value import (
+    AssetEntity,
+    AssetFields,
+    ClassEntity,
+    ContainerEntity,
+    DataModelEntity,
+    DMSEntity,
+    DMSNodeEntity,
+    DMSUnknownEntity,
+    DMSVersionedEntity,
+    EdgeEntity,
+    Entity,
+    ReferenceEntity,
+    RelationshipEntity,
+    ReverseConnectionEntity,
+    T_Entity,
+    UnitEntity,
+    UnknownEntity,
+    ViewEntity,
+)
+from ._types import CdfResourceEntityList, ClassEntityList, ContainerEntityList, URLEntity, ViewEntityList
+from ._wrapped import DMSFilter, HasDataFilter, NodeTypeFilter, RawFilter, WrappedEntity
+
+__all__ = [
+    "Entity",
+    "UnitEntity",
+    "DMSVersionedEntity",
+    "RelationshipEntity",
+    "EdgeEntity",
+    "ViewEntity",
+    "ClassEntity",
+    "ViewEntityList",
+    "UnknownEntity",
+    "EntityTypes",
+    "Unknown",
+    "Undefined",
+    "T_Entity",
+    "AssetEntity",
+    "AssetFields",
+    "ReverseConnectionEntity",
+    "MultiValueTypeInfo",
+    "DMSNodeEntity",
+    "DMSEntity",
+    "ContainerEntity",
+    "DMSUnknownEntity",
+    "DataModelEntity",
+    "ReferenceEntity",
+    "ClassEntityList",
+    "CdfResourceEntityList",
+    "ContainerEntityList",
+    "URLEntity",
+    "load_value_type",
+    "load_dms_value_type",
+    "load_connection",
+    "WrappedEntity",
+    "NodeTypeFilter",
+    "DMSFilter",
+    "HasDataFilter",
+    "RawFilter",
+]

cognite/neat/rules/models/entities/_constants.py

@@ -0,0 +1,73 @@
+import re
+import sys
+
+from pydantic import BaseModel
+
+if sys.version_info >= (3, 11):
+    from enum import StrEnum
+else:
+    from backports.strenum import StrEnum
+
+
+class EntityTypes(StrEnum):
+    view_non_versioned = "view_non_versioned"
+    subject = "subject"
+    predicate = "predicate"
+    object = "object"
+    class_ = "class"
+    parent_class = "parent_class"
+    property_ = "property"
+    object_property = "ObjectProperty"
+    data_property = "DatatypeProperty"
+    annotation_property = "AnnotationProperty"
+    object_value_type = "object_value_type"
+    data_value_type = "data_value_type"  # these are strings, floats, ...
+    xsd_value_type = "xsd_value_type"
+    dms_value_type = "dms_value_type"
+    dms_node = "dms_node"
+    view = "view"
+    reference_entity = "reference_entity"
+    container = "container"
+    datamodel = "datamodel"
+    undefined = "undefined"
+    multi_value_type = "multi_value_type"
+    asset = "asset"
+    relationship = "relationship"
+    edge = "edge"
+    reverse = "reverse"
+    unit = "unit"
+
+
+# ALLOWED
+_ALLOWED_PATTERN = r"[^a-zA-Z0-9-_.]"
+
+# FOR PARSING STRINGS:
+_PREFIX_REGEX = r"[a-zA-Z]+[a-zA-Z0-9-_.]*[a-zA-Z0-9]+"
+_SUFFIX_REGEX = r"[a-zA-Z0-9-_.]+[a-zA-Z0-9]|[-_.]*[a-zA-Z0-9]+"
+_VERSION_REGEX = r"[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])?"
+_PROPERTY_REGEX = r"[a-zA-Z0-9][a-zA-Z0-9_-]*[a-zA-Z0-9]?"
+_ENTITY_ID_REGEX = rf"{_PREFIX_REGEX}:({_SUFFIX_REGEX})"
+_ENTITY_ID_REGEX_COMPILED = re.compile(rf"^(?P<prefix>{_PREFIX_REGEX}):(?P<suffix>{_SUFFIX_REGEX})$")
+_VERSIONED_ENTITY_REGEX_COMPILED = re.compile(
+    rf"^(?P<prefix>{_PREFIX_REGEX}):(?P<suffix>{_SUFFIX_REGEX})\(version=(?P<version>{_VERSION_REGEX})\)$"
+)
+_CLASS_ID_REGEX = rf"(?P<{EntityTypes.class_}>{_ENTITY_ID_REGEX})"
+_CLASS_ID_REGEX_COMPILED = re.compile(rf"^{_CLASS_ID_REGEX}$")
+_PROPERTY_ID_REGEX = rf"\((?P<{EntityTypes.property_}>{_ENTITY_ID_REGEX})\)"
+
+ENTITY_PATTERN = re.compile(r"^(?P<prefix>.*?):?(?P<suffix>[^(:]*)(\((?P<content>.+)\))?$")
+MULTI_VALUE_TYPE_PATTERN = re.compile(r"^(?P<types>.*?)(\((?P<content>[^)]+)\))?$")
+
+
+class _UndefinedType(BaseModel): ...
+
+
+class _UnknownType(BaseModel):
+    def __str__(self) -> str:
+        return "#N/A"
+
+
+# This is a trick to make Undefined and Unknown singletons
+Undefined = _UndefinedType()
+Unknown = _UnknownType()
+_PARSE = object()

cognite/neat/rules/models/entities/_loaders.py

@@ -0,0 +1,76 @@
+from typing import Literal
+
+from cognite.neat.issues.errors import NeatTypeError
+from cognite.neat.rules.models.data_types import DataType
+
+from ._multi_value import MultiValueTypeInfo
+from ._single_value import (
+    ClassEntity,
+    DMSUnknownEntity,
+    EdgeEntity,
+    ReverseConnectionEntity,
+    Unknown,
+    UnknownEntity,
+    ViewEntity,
+)
+
+
+def load_value_type(
+    raw: str | MultiValueTypeInfo | DataType | ClassEntity | UnknownEntity, default_prefix: str
+) -> MultiValueTypeInfo | DataType | ClassEntity | UnknownEntity:
+    if isinstance(raw, MultiValueTypeInfo | DataType | ClassEntity | UnknownEntity):
+        return raw
+    elif isinstance(raw, str):
+        # property holding xsd data type
+        # check if it is multi value type
+        if "|" in raw:
+            value_type = MultiValueTypeInfo.load(raw)
+            value_type.set_default_prefix(default_prefix)
+            return value_type
+        elif DataType.is_data_type(raw):
+            return DataType.load(raw)
+
+        # unknown value type
+        elif raw == str(Unknown):
+            return UnknownEntity()
+
+        # property holding link to class
+        else:
+            return ClassEntity.load(raw, prefix=default_prefix)
+    else:
+        raise NeatTypeError(f"Invalid value type: {type(raw)}")
+
+
+def load_dms_value_type(
+    raw: str | DataType | ViewEntity | DMSUnknownEntity,
+    default_space: str,
+    default_version: str,
+) -> DataType | ViewEntity | DMSUnknownEntity:
+    if isinstance(raw, DataType | ViewEntity | DMSUnknownEntity):
+        return raw
+    elif isinstance(raw, str):
+        if DataType.is_data_type(raw):
+            return DataType.load(raw)
+        elif raw == str(Unknown):
+            return DMSUnknownEntity()
+        else:
+            return ViewEntity.load(raw, space=default_space, version=default_version)
+    raise NeatTypeError(f"Invalid value type: {type(raw)}")
+
+
+def load_connection(
+    raw: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | str | None,
+    default_space: str,
+    default_version: str,
+) -> Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None:
+    if (
+        isinstance(raw, EdgeEntity | ReverseConnectionEntity)
+        or raw is None
+        or (isinstance(raw, str) and raw == "direct")
+    ):
+        return raw  # type: ignore[return-value]
+    elif isinstance(raw, str) and raw.startswith("edge"):
+        return EdgeEntity.load(raw, space=default_space, version=default_version)  # type: ignore[return-value]
+    elif isinstance(raw, str) and raw.startswith("reverse"):
+        return ReverseConnectionEntity.load(raw)  # type: ignore[return-value]
+    raise NeatTypeError(f"Invalid connection: {type(raw)}")

cognite/neat/rules/models/entities/_multi_value.py

@@ -0,0 +1,67 @@
+from typing import Any, ClassVar
+
+from pydantic import (
+    BaseModel,
+    model_serializer,
+    model_validator,
+)
+
+from cognite.neat.rules.models.data_types import DataType
+
+from ._constants import _PARSE, EntityTypes, Undefined
+from ._single_value import ClassEntity, UnknownEntity
+
+
+class MultiValueTypeInfo(BaseModel):
+    type_: ClassVar[EntityTypes] = EntityTypes.multi_value_type
+    types: list[DataType | ClassEntity]
+
+    def __str__(self) -> str:
+        return " | ".join([str(t) for t in self.types])
+
+    @model_serializer(when_used="unless-none", return_type=str)
+    def as_str(self) -> str:
+        return str(self)
+
+    @classmethod
+    def load(cls, data: Any) -> "MultiValueTypeInfo":
+        # already instance of MultiValueTypeInfo
+        if isinstance(data, cls):
+            return data
+
+        # it is a raw string that needs to be parsed
+        elif isinstance(data, str):
+            return cls.model_validate({_PARSE: data})
+
+        # it is dict that needs to be parsed
+        else:
+            return cls.model_validate(data)
+
+    @model_validator(mode="before")
+    def _load(cls, data: Any) -> "dict | MultiValueTypeInfo":
+        if isinstance(data, dict) and _PARSE in data:
+            data = data[_PARSE]
+        elif isinstance(data, dict):
+            return data
+        else:
+            raise ValueError(f"Cannot load {cls.__name__} from {data}")
+
+        result = cls._parse(data)
+        return result
+
+    @classmethod
+    def _parse(cls, raw: str) -> dict:
+        if not (types := [type_.strip() for type_ in raw.split("|")]):
+            return {"types": [UnknownEntity()]}
+        else:
+            return {
+                "types": [
+                    (DataType.load(type_) if DataType.is_data_type(type_) else ClassEntity.load(type_))
+                    for type_ in types
+                ]
+            }
+
+    def set_default_prefix(self, prefix: str):
+        for type_ in self.types:
+            if isinstance(type_, ClassEntity) and type_.prefix is Undefined:
+                type_.prefix = prefix