cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl

cognite/neat/_rules/models/dms/_validation.py

@@ -3,11 +3,11 @@ from typing import Any, ClassVar, cast
 
 from cognite.client import data_modeling as dm
 
+from cognite.neat._client.data_classes.schema import DMSSchema
 from cognite.neat._constants import COGNITE_MODELS, DMS_CONTAINER_PROPERTY_SIZE_LIMIT
 from cognite.neat._issues import IssueList, NeatError, NeatIssue, NeatIssueList
 from cognite.neat._issues.errors import (
     PropertyDefinitionDuplicatedError,
-    ResourceChangedError,
     ResourceNotDefinedError,
 )
 from cognite.neat._issues.errors._properties import ReversedConnectionNotFeasibleError
@@ -20,7 +20,7 @@ from cognite.neat._issues.warnings.user_modeling import (
     NotNeatSupportedFilterWarning,
     ViewPropertyLimitWarning,
 )
-from cognite.neat._rules.models._base_rules import DataModelType, ExtensionCategory, SchemaCompleteness
+from cognite.neat._rules.analysis import DMSAnalysis
 from cognite.neat._rules.models.data_types import DataType
 from cognite.neat._rules.models.entities import ContainerEntity, RawFilter
 from cognite.neat._rules.models.entities._single_value import (
@@ -29,7 +29,6 @@ from cognite.neat._rules.models.entities._single_value import (
 )
 
 from ._rules import DMSProperty, DMSRules
-from ._schema import DMSSchema
 
 
 class DMSPostValidation:
@@ -47,6 +46,7 @@ class DMSPostValidation:
         self.containers = rules.containers
         self.views = rules.views
         self.issue_list = IssueList()
+        self.probe = DMSAnalysis(rules)
 
     def validate(self) -> NeatIssueList:
         self._validate_raw_filter()
@@ -55,12 +55,7 @@ class DMSPostValidation:
         self._validate_reverse_connections()
 
         self._referenced_views_and_containers_are_existing_and_proper_size()
-        if self.metadata.schema_ is SchemaCompleteness.extended:
-            self._validate_extension()
-        if self.metadata.schema_ is SchemaCompleteness.partial:
-            return self.issue_list
         dms_schema = self.rules.as_schema()
-        self.issue_list.extend(dms_schema.validate())
         self._validate_performance(dms_schema)
         return self.issue_list
 
@@ -160,9 +155,10 @@ class DMSPostValidation:
         self.issue_list.extend(errors)
 
     def _referenced_views_and_containers_are_existing_and_proper_size(self) -> None:
+        # TODO: Split this method and keep only validation that should be independent of
+        # whether view and/or container exist in the pydantic model instance
+        # other validation should be done through NeatSession.verify()
         defined_views = {view.view.as_id() for view in self.views}
-        if self.metadata.schema_ is SchemaCompleteness.extended and self.rules.last:
-            defined_views |= {view.view.as_id() for view in self.rules.last.views}
 
         property_count_by_view: dict[dm.ViewId, int] = defaultdict(int)
         errors: list[NeatIssue] = []
@@ -170,7 +166,7 @@ class DMSPostValidation:
             view_id = prop.view.as_id()
             if view_id not in defined_views:
                 errors.append(
-                    ResourceNotDefinedError[dm.ViewId](
+                    ResourceNotDefinedError(
                         identifier=view_id,
                         resource_type="view",
                         location="Views Sheet",
@@ -184,103 +180,9 @@ class DMSPostValidation:
         for view_id, count in property_count_by_view.items():
             if count > DMS_CONTAINER_PROPERTY_SIZE_LIMIT:
                 errors.append(ViewPropertyLimitWarning(view_id, count))
-        if self.metadata.schema_ is SchemaCompleteness.complete:
-            defined_containers = {container.container.as_id() for container in self.containers or []}
-            if self.metadata.data_model_type == DataModelType.solution and self.rules.reference:
-                defined_containers |= {
-                    container.container.as_id() for container in self.rules.reference.containers or []
-                }
 
-            for prop_no, prop in enumerate(self.properties):
-                if prop.container and (container_id := prop.container.as_id()) not in defined_containers:
-                    errors.append(
-                        ResourceNotDefinedError(
-                            identifier=container_id,
-                            resource_type="container",
-                            location="Containers Sheet",
-                            column_name="Container",
-                            row_number=prop_no,
-                            sheet_name="Properties",
-                        )
-                    )
-            for _container_no, container in enumerate(self.containers or []):
-                for constraint_no, constraint in enumerate(container.constraint or []):
-                    if constraint.as_id() not in defined_containers:
-                        errors.append(
-                            ResourceNotDefinedError(
-                                identifier=constraint.as_id(),
-                                resource_type="container",
-                                location="Containers Sheet",
-                                column_name="Constraint",
-                                row_number=constraint_no,
-                                sheet_name="Properties",
-                            )
-                        )
         self.issue_list.extend(errors)
 
-    def _validate_extension(self) -> None:
-        if self.metadata.schema_ is not SchemaCompleteness.extended:
-            return None
-        if not self.rules.last:
-            raise ValueError("The schema is set to 'extended', but no last rules are provided to validate against")
-        if self.metadata.extension is ExtensionCategory.rebuild:
-            # Everything is allowed
-            return None
-        user_schema = self.rules.as_schema()
-        new_containers = user_schema.containers.copy()
-
-        last_schema = self.rules.last.as_schema()
-        existing_containers = last_schema.containers.copy()
-
-        for container_id, container in new_containers.items():
-            existing_container = existing_containers.get(container_id)
-            if not existing_container or existing_container == container:
-                # No problem
-                continue
-            new_dumped = container.dump()
-            existing_dumped = existing_container.dump()
-            changed_attributes, changed_properties = self._changed_attributes_and_properties(
-                new_dumped, existing_dumped
-            )
-            self.issue_list.append(
-                ResourceChangedError(
-                    container_id,
-                    "container",
-                    changed_properties=frozenset(changed_properties),
-                    changed_attributes=frozenset(changed_attributes),
-                )
-            )
-
-        if self.metadata.extension is ExtensionCategory.reshape:
-            # Reshape allows changes to views
-            return None
-
-        new_views = user_schema.views.copy()
-        existing_views = last_schema.views.copy()
-        for view_id, view in new_views.items():
-            existing_view = existing_views.get(view_id)
-            if not existing_view or existing_view == view:
-                # No problem
-                continue
-            changed_attributes, changed_properties = self._changed_attributes_and_properties(
-                view.dump(), existing_view.dump()
-            )
-            existing_properties = existing_view.properties or {}
-            changed_properties = [prop for prop in changed_properties if prop in existing_properties]
-            changed_attributes = [attr for attr in changed_attributes if attr not in self.changeable_view_attributes]
-
-            if not changed_attributes and not changed_properties:
-                # Only added new properties, no problem
-                continue
-            self.issue_list.append(
-                ResourceChangedError(
-                    view_id,
-                    "view",
-                    changed_properties=frozenset(changed_properties),
-                    changed_attributes=frozenset(changed_attributes),
-                )
-            )
-
     def _validate_performance(self, dms_schema: DMSSchema) -> None:
         for view_id, view in dms_schema.views.items():
             mapped_containers = dms_schema._get_mapped_container_from_view(view_id)
@@ -320,7 +222,7 @@ class DMSPostValidation:
                     UndefinedViewWarning(
                         str(prop_.view),
                         str(prop_.value_type),
-                        prop_.property_,
+                        prop_.view_property,
                     )
                 )
 
@@ -329,7 +231,12 @@ class DMSPostValidation:
         if self.metadata.as_data_model_id() in COGNITE_MODELS:
             return None
 
-        properties_by_ids = {f"{prop_.view!s}.{prop_.property_}": prop_ for prop_ in self.properties}
+        properties_by_ids = {
+            f"{prop_.view!s}.{prop_.view_property}": prop_
+            for properties in self.probe.classes_with_properties(True, True).values()
+            for prop_ in properties
+        }
+
         reversed_by_ids = {
             id_: prop_
             for id_, prop_ in properties_by_ids.items()
@@ -343,7 +250,7 @@ class DMSPostValidation:
                     ReversedConnectionNotFeasibleError(
                         id_,
                         "reversed connection",
-                        prop_.property_,
+                        prop_.view_property,
                         str(prop_.view),
                         str(prop_.value_type),
                         cast(ReverseConnectionEntity, prop_.connection).property_,
@@ -355,7 +262,7 @@ class DMSPostValidation:
                     ReversedConnectionNotFeasibleError(
                         id_,
                         "view property",
-                        prop_.property_,
+                        prop_.view_property,
                         str(prop_.view),
                         str(prop_.value_type),
                         cast(ReverseConnectionEntity, prop_.connection).property_,

cognite/neat/_rules/models/entities/_single_value.py

@@ -3,7 +3,7 @@ import sys
 from abc import ABC, abstractmethod
 from functools import total_ordering
 from types import UnionType
-from typing import Any, ClassVar, Generic, Literal, TypeVar, Union, cast, get_args, get_origin
+from typing import Any, ClassVar, Generic, Literal, TypeVar, Union, cast, get_args, get_origin, overload
 
 from cognite.client.data_classes.data_modeling import DirectRelationReference
 from cognite.client.data_classes.data_modeling.data_types import UnitReference
@@ -22,6 +22,7 @@ from pydantic import (
     model_validator,
 )
 
+from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._utils.text import replace_non_alphanumeric_with_underscore
 
 if sys.version_info >= (3, 11):
@@ -56,14 +57,29 @@ class Entity(BaseModel, extra="ignore"):
     suffix: str
 
     @classmethod
-    def load(cls: "type[T_Entity]", data: Any, **defaults) -> "T_Entity | UnknownEntity":
+    @overload
+    def load(cls: "type[T_Entity]", data: Any, strict: Literal[True], **defaults) -> "T_Entity": ...
+
+    @classmethod
+    @overload
+    def load(
+        cls: "type[T_Entity]", data: Any, strict: Literal[False] = False, **defaults
+    ) -> "T_Entity | UnknownEntity": ...
+
+    @classmethod
+    def load(cls: "type[T_Entity]", data: Any, strict: bool = False, **defaults) -> "T_Entity | UnknownEntity":
         if isinstance(data, cls):
             return data
         elif isinstance(data, str) and data == str(Unknown):
+            if strict:
+                raise NeatValueError(f"Failed to load entity {data!s}")
             return UnknownEntity(prefix=Undefined, suffix=Unknown)
         if defaults and isinstance(defaults, dict):
             # This is a trick to pass in default values
-            return cls.model_validate({_PARSE: data, "defaults": defaults})
+            try:
+                return cls.model_validate({_PARSE: data, "defaults": defaults})
+            except ValueError:
+                raise
         else:
             return cls.model_validate(data)
 
@@ -291,9 +307,21 @@ class DMSEntity(Entity, Generic[T_ID], ABC):
             defaults["prefix"] = defaults.pop("space")
         return super().dump(**defaults)
 
+    @classmethod  # type: ignore[override]
+    @overload
+    def load(cls: "type[T_DMSEntity]", data: Any, strict: Literal[True], **defaults) -> "T_DMSEntity": ...
+
+    @classmethod
+    @overload
+    def load(
+        cls: "type[T_DMSEntity]", data: Any, strict: Literal[False] = False, **defaults
+    ) -> "T_DMSEntity | DMSUnknownEntity": ...
+
     @classmethod
-    def load(cls: "type[T_DMSEntity]", data: Any, **defaults) -> "T_DMSEntity | DMSUnknownEntity":  # type: ignore[override]
+    def load(cls: "type[T_DMSEntity]", data: Any, strict: bool = False, **defaults) -> "T_DMSEntity | DMSUnknownEntity":  # type: ignore[override]
         if isinstance(data, str) and data == str(Unknown):
+            if strict:
+                raise NeatValueError(f"Failed to load entity {data!s}")
             return DMSUnknownEntity.from_id(None)
         return cast(T_DMSEntity, super().load(data, **defaults))
 

cognite/neat/_rules/models/information/_rules.py

@@ -1,7 +1,6 @@
 import math
 import sys
 from collections.abc import Hashable
-from datetime import datetime
 from typing import TYPE_CHECKING, Any, ClassVar
 
 import pandas as pd
@@ -9,20 +8,14 @@ from pydantic import Field, field_serializer, field_validator, model_validator
 from pydantic_core.core_schema import SerializationInfo
 from rdflib import Namespace, URIRef
 
-from cognite.neat._constants import DEFAULT_NAMESPACE, get_default_prefixes
+from cognite.neat._constants import get_default_prefixes
 from cognite.neat._issues.errors import NeatValueError, PropertyDefinitionError
 from cognite.neat._rules._constants import EntityTypes
 from cognite.neat._rules.models._base_rules import (
     BaseMetadata,
     BaseRules,
-    ClassRef,
-    DataModelType,
-    ExtensionCategory,
-    ExtensionCategoryType,
-    MatchType,
-    PropertyRef,
+    DataModelAspect,
     RoleTypes,
-    SchemaCompleteness,
     SheetList,
     SheetRow,
 )
@@ -35,10 +28,6 @@ from cognite.neat._rules.models._types import (
     ClassEntityType,
     InformationPropertyType,
     MultiValueTypeType,
-    NamespaceType,
-    PrefixType,
-    StrListType,
-    VersionType,
 )
 from cognite.neat._rules.models.data_types import DataType
 from cognite.neat._rules.models.entities import (
@@ -46,10 +35,8 @@ from cognite.neat._rules.models.entities import (
     ClassEntityList,
     Entity,
     MultiValueTypeInfo,
-    ReferenceEntity,
     Undefined,
     UnknownEntity,
-    URLEntity,
 )
 
 if TYPE_CHECKING:
@@ -64,62 +51,11 @@ else:
 
 class InformationMetadata(BaseMetadata):
     role: ClassVar[RoleTypes] = RoleTypes.information
-    data_model_type: DataModelType = Field(DataModelType.enterprise, alias="dataModelType")
-    schema_: SchemaCompleteness = Field(SchemaCompleteness.partial, alias="schema")
-    extension: ExtensionCategoryType | None = ExtensionCategory.addition
-
-    prefix: PrefixType
-    namespace: NamespaceType
-
-    name: str = Field(
-        alias="title",
-        description="Human readable name of the data model",
-        min_length=1,
-        max_length=255,
-    )
-    description: str | None = Field(None, min_length=1, max_length=1024)
-    version: VersionType
-
-    created: datetime = Field(
-        description=("Date of the data model creation"),
-    )
-
-    updated: datetime = Field(
-        description=("Date of the data model update"),
-    )
-    creator: StrListType = Field(
-        description=(
-            "List of contributors to the data model creation, "
-            "typically information architects are considered as contributors."
-        ),
-    )
-    license: str | None = None
-    rights: str | None = None
-
-    @model_validator(mode="after")
-    def extension_none_but_schema_extend(self) -> Self:
-        if self.extension is None:
-            self.extension = ExtensionCategory.addition
-            return self
-        return self
-
-    @field_validator("schema_", mode="plain")
-    def as_enum_schema(cls, value: str) -> SchemaCompleteness:
-        return SchemaCompleteness(value.strip())
-
-    @field_validator("extension", mode="plain")
-    def as_enum_extension(cls, value: str) -> ExtensionCategory:
-        return ExtensionCategory(value.strip())
+    aspect: ClassVar[DataModelAspect] = DataModelAspect.logical
 
-    @field_validator("data_model_type", mode="plain")
-    def as_enum_model_type(cls, value: str) -> DataModelType:
-        return DataModelType(value.strip())
-
-    def as_identifier(self) -> str:
-        return f"{self.prefix}:{self.name}"
-
-    def get_prefix(self) -> str:
-        return self.prefix
+    # Linking to Conceptual and Physical data model aspects
+    physical: URIRef | None = Field(None, description="Link to the logical data model aspect")
+    conceptual: URIRef | None = Field(None, description="Link to the logical data model aspect")
 
 
 def _get_metadata(context: Any) -> InformationMetadata | None:
@@ -135,48 +71,36 @@ class InformationClass(SheetRow):
     Args:
         class_: The class ID of the class.
         description: A description of the class.
-        parent: The parent class of the class.
-        reference: Reference of the source of the information for given resource
-        match_type: The match type of the resource being described and the source entity.
+        implements: Which classes the current class implements.
     """
 
     class_: ClassEntityType = Field(alias="Class")
     name: str | None = Field(alias="Name", default=None)
     description: str | None = Field(alias="Description", default=None)
-    parent: ClassEntityList | None = Field(alias="Parent Class", default=None)
-    reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
-    match_type: MatchType | None = Field(alias="Match Type", default=None)
-    comment: str | None = Field(alias="Comment", default=None)
+    implements: ClassEntityList | None = Field(alias="Implements", default=None)
 
     def _identifier(self) -> tuple[Hashable, ...]:
         return (self.class_,)
 
-    @field_serializer("reference", when_used="always")
-    def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
-        if isinstance(info.context, dict) and info.context.get("as_reference") is True:
-            return self.class_.dump()
-        return str(value) if value is not None else None
-
     @field_serializer("class_", when_used="unless-none")
     def remove_default_prefix(self, value: Any, info: SerializationInfo) -> str:
         if (metadata := _get_metadata(info.context)) and isinstance(value, Entity):
             return value.dump(prefix=metadata.prefix, version=metadata.version)
         return str(value)
 
-    @field_serializer("parent", when_used="unless-none")
+    @field_serializer("implements", when_used="unless-none")
     def remove_default_prefixes(self, value: Any, info: SerializationInfo) -> str:
         if isinstance(value, list) and (metadata := _get_metadata(info.context)):
             return ",".join(
-                parent.dump(prefix=metadata.prefix, version=metadata.version)
-                if isinstance(parent, Entity)
-                else str(parent)
-                for parent in value
+                (
+                    class_.dump(prefix=metadata.prefix, version=metadata.version)
+                    if isinstance(class_, Entity)
+                    else str(class_)
+                )
+                for class_ in value
             )
         return ",".join(str(value) for value in value)
 
-    def as_reference(self) -> ClassRef:
-        return ClassRef(Class=self.class_)
-
 
 class InformationProperty(SheetRow):
     """
@@ -191,8 +115,6 @@ class InformationProperty(SheetRow):
         min_count: Minimum count of the property values. Defaults to 0
         max_count: Maximum count of the property values. Defaults to None
         default: Default value of the property
-        reference: Reference to the source of the information, HTTP URI
-        match_type: The match type of the resource being described and the source entity.
         transformation: Actual rule for the transformation from source to target representation of
               knowledge graph. Defaults to None (no transformation)
     """
@@ -207,10 +129,7 @@ class InformationProperty(SheetRow):
     min_count: int | None = Field(alias="Min Count", default=None)
     max_count: int | float | None = Field(alias="Max Count", default=None)
     default: Any | None = Field(alias="Default", default=None)
-    reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
-    match_type: MatchType | None = Field(alias="Match Type", default=None)
     transformation: RDFPath | None = Field(alias="Transformation", default=None)
-    comment: str | None = Field(alias="Comment", default=None)
     inherited: bool = Field(
         default=False,
         exclude=True,
@@ -281,19 +200,6 @@ class InformationProperty(SheetRow):
             return None
         return value
 
-    @field_serializer("reference", when_used="always")
-    def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
-        # When rules as dumped as reference, we set the reference to the class
-        if isinstance(info.context, dict) and info.context.get("as_reference") is True:
-            return str(
-                ReferenceEntity(
-                    prefix=str(self.class_.prefix),
-                    suffix=self.class_.suffix,
-                    property=self.property_,
-                )
-            )
-        return str(value) if value is not None else None
-
     @field_serializer("class_", "value_type", when_used="unless-none")
     def remove_default_prefix(self, value: Any, info: SerializationInfo) -> str:
         if (metadata := _get_metadata(info.context)) and isinstance(value, Entity):
@@ -322,17 +228,12 @@ class InformationProperty(SheetRow):
             isinstance(self.max_count, int | float) and self.max_count > 1
         )
 
-    def as_reference(self) -> PropertyRef:
-        return PropertyRef(Class=self.class_, Property=self.property_)
-
 
 class InformationRules(BaseRules):
     metadata: InformationMetadata = Field(alias="Metadata")
     properties: SheetList[InformationProperty] = Field(alias="Properties")
     classes: SheetList[InformationClass] = Field(alias="Classes")
     prefixes: dict[str, Namespace] = Field(default_factory=get_default_prefixes, alias="Prefixes")
-    last: "InformationRules | None" = Field(None, alias="Last")
-    reference: "InformationRules | None" = Field(None, alias="Reference")
 
     @field_validator("prefixes", mode="before")
     def parse_str(cls, values: Any) -> Any:
@@ -355,10 +256,10 @@ class InformationRules(BaseRules):
             if property_.class_.prefix is Undefined:
                 property_.class_.prefix = self.metadata.prefix
 
-        # update parent classes
+        # update implements
         for class_ in self.classes:
-            if class_.parent:
-                for parent in class_.parent:
+            if class_.implements:
+                for parent in class_.implements:
                     if not isinstance(parent.prefix, str):
                         parent.prefix = self.metadata.prefix
             if class_.class_.prefix is Undefined:
@@ -387,14 +288,10 @@ class InformationRules(BaseRules):
             "type": "Logical Data Model",
             "intended for": "Information Architect",
             "name": self.metadata.name,
-            "external_id": self.metadata.prefix,
+            "external_id": self.metadata.external_id,
             "version": self.metadata.version,
             "classes": len(self.classes),
             "properties": len(self.properties),
         }
 
         return pd.DataFrame([summary]).T.rename(columns={0: ""})._repr_html_()  # type: ignore
-
-    @property
-    def id_(self) -> URIRef:
-        return DEFAULT_NAMESPACE[f"data-model/verified/info/{self.metadata.prefix}/{self.metadata.version}"]