cognite-neat 0.97.2__py3-none-any.whl → 0.98.0__py3-none-any.whl

cognite/neat/_rules/models/dms/_validation.py

@@ -7,7 +7,6 @@ from cognite.neat._constants import COGNITE_MODELS, DMS_CONTAINER_PROPERTY_SIZE_
 from cognite.neat._issues import IssueList, NeatError, NeatIssue, NeatIssueList
 from cognite.neat._issues.errors import (
     PropertyDefinitionDuplicatedError,
-    ResourceChangedError,
     ResourceNotDefinedError,
 )
 from cognite.neat._issues.errors._properties import ReversedConnectionNotFeasibleError
@@ -20,7 +19,6 @@ from cognite.neat._issues.warnings.user_modeling import (
     NotNeatSupportedFilterWarning,
     ViewPropertyLimitWarning,
 )
-from cognite.neat._rules.models._base_rules import DataModelType, ExtensionCategory, SchemaCompleteness
 from cognite.neat._rules.models.data_types import DataType
 from cognite.neat._rules.models.entities import ContainerEntity, RawFilter
 from cognite.neat._rules.models.entities._single_value import (
@@ -55,12 +53,7 @@ class DMSPostValidation:
         self._validate_reverse_connections()
 
         self._referenced_views_and_containers_are_existing_and_proper_size()
-        if self.metadata.schema_ is SchemaCompleteness.extended:
-            self._validate_extension()
-        if self.metadata.schema_ is SchemaCompleteness.partial:
-            return self.issue_list
         dms_schema = self.rules.as_schema()
-        self.issue_list.extend(dms_schema.validate())
         self._validate_performance(dms_schema)
         return self.issue_list
 
@@ -157,27 +150,13 @@ class DMSPostValidation:
                     )
                 )
 
-            # This sets the container definition for all the properties where it is not defined.
-            # This allows the user to define the container only once.
-            value_type = next(iter(value_types))
-            list_definition = next(iter(list_definitions)) if list_definitions else None
-            nullable_definition = next(iter(nullable_definitions)) if nullable_definitions else None
-            default_definition = next(iter(default_definitions)) if default_definitions else None
-            index_definition = next(iter(index_definitions)).split(",") if index_definitions else None
-            constraint_definition = next(iter(constraint_definitions)).split(",") if constraint_definitions else None
-            for _, prop in properties:
-                prop.value_type = value_type
-                prop.is_list = prop.is_list or list_definition
-                prop.nullable = prop.nullable or nullable_definition
-                prop.default = prop.default or default_definition
-                prop.index = prop.index or index_definition
-                prop.constraint = prop.constraint or constraint_definition
         self.issue_list.extend(errors)
 
     def _referenced_views_and_containers_are_existing_and_proper_size(self) -> None:
+        # TODO: Split this method and keep only validation that should be independent of
+        # whether view and/or container exist in the pydantic model instance
+        # other validation should be done through NeatSession.verify()
         defined_views = {view.view.as_id() for view in self.views}
-        if self.metadata.schema_ is SchemaCompleteness.extended and self.rules.last:
-            defined_views |= {view.view.as_id() for view in self.rules.last.views}
 
         property_count_by_view: dict[dm.ViewId, int] = defaultdict(int)
         errors: list[NeatIssue] = []
@@ -199,103 +178,9 @@ class DMSPostValidation:
         for view_id, count in property_count_by_view.items():
             if count > DMS_CONTAINER_PROPERTY_SIZE_LIMIT:
                 errors.append(ViewPropertyLimitWarning(view_id, count))
-        if self.metadata.schema_ is SchemaCompleteness.complete:
-            defined_containers = {container.container.as_id() for container in self.containers or []}
-            if self.metadata.data_model_type == DataModelType.solution and self.rules.reference:
-                defined_containers |= {
-                    container.container.as_id() for container in self.rules.reference.containers or []
-                }
 
-            for prop_no, prop in enumerate(self.properties):
-                if prop.container and (container_id := prop.container.as_id()) not in defined_containers:
-                    errors.append(
-                        ResourceNotDefinedError(
-                            identifier=container_id,
-                            resource_type="container",
-                            location="Containers Sheet",
-                            column_name="Container",
-                            row_number=prop_no,
-                            sheet_name="Properties",
-                        )
-                    )
-            for _container_no, container in enumerate(self.containers or []):
-                for constraint_no, constraint in enumerate(container.constraint or []):
-                    if constraint.as_id() not in defined_containers:
-                        errors.append(
-                            ResourceNotDefinedError(
-                                identifier=constraint.as_id(),
-                                resource_type="container",
-                                location="Containers Sheet",
-                                column_name="Constraint",
-                                row_number=constraint_no,
-                                sheet_name="Properties",
-                            )
-                        )
         self.issue_list.extend(errors)
 
-    def _validate_extension(self) -> None:
-        if self.metadata.schema_ is not SchemaCompleteness.extended:
-            return None
-        if not self.rules.last:
-            raise ValueError("The schema is set to 'extended', but no last rules are provided to validate against")
-        if self.metadata.extension is ExtensionCategory.rebuild:
-            # Everything is allowed
-            return None
-        user_schema = self.rules.as_schema()
-        new_containers = user_schema.containers.copy()
-
-        last_schema = self.rules.last.as_schema()
-        existing_containers = last_schema.containers.copy()
-
-        for container_id, container in new_containers.items():
-            existing_container = existing_containers.get(container_id)
-            if not existing_container or existing_container == container:
-                # No problem
-                continue
-            new_dumped = container.dump()
-            existing_dumped = existing_container.dump()
-            changed_attributes, changed_properties = self._changed_attributes_and_properties(
-                new_dumped, existing_dumped
-            )
-            self.issue_list.append(
-                ResourceChangedError(
-                    container_id,
-                    "container",
-                    changed_properties=frozenset(changed_properties),
-                    changed_attributes=frozenset(changed_attributes),
-                )
-            )
-
-        if self.metadata.extension is ExtensionCategory.reshape:
-            # Reshape allows changes to views
-            return None
-
-        new_views = user_schema.views.copy()
-        existing_views = last_schema.views.copy()
-        for view_id, view in new_views.items():
-            existing_view = existing_views.get(view_id)
-            if not existing_view or existing_view == view:
-                # No problem
-                continue
-            changed_attributes, changed_properties = self._changed_attributes_and_properties(
-                view.dump(), existing_view.dump()
-            )
-            existing_properties = existing_view.properties or {}
-            changed_properties = [prop for prop in changed_properties if prop in existing_properties]
-            changed_attributes = [attr for attr in changed_attributes if attr not in self.changeable_view_attributes]
-
-            if not changed_attributes and not changed_properties:
-                # Only added new properties, no problem
-                continue
-            self.issue_list.append(
-                ResourceChangedError(
-                    view_id,
-                    "view",
-                    changed_properties=frozenset(changed_properties),
-                    changed_attributes=frozenset(changed_attributes),
-                )
-            )
-
     def _validate_performance(self, dms_schema: DMSSchema) -> None:
         for view_id, view in dms_schema.views.items():
             mapped_containers = dms_schema._get_mapped_container_from_view(view_id)
@@ -335,7 +220,7 @@ class DMSPostValidation:
                     UndefinedViewWarning(
                         str(prop_.view),
                         str(prop_.value_type),
-                        prop_.property_,
+                        prop_.view_property,
                     )
                 )
 
@@ -344,7 +229,7 @@ class DMSPostValidation:
         if self.metadata.as_data_model_id() in COGNITE_MODELS:
             return None
 
-        properties_by_ids = {f"{prop_.view!s}.{prop_.property_}": prop_ for prop_ in self.properties}
+        properties_by_ids = {f"{prop_.view!s}.{prop_.view_property}": prop_ for prop_ in self.properties}
         reversed_by_ids = {
             id_: prop_
             for id_, prop_ in properties_by_ids.items()
@@ -354,11 +239,12 @@ class DMSPostValidation:
         for id_, prop_ in reversed_by_ids.items():
             source_id = f"{prop_.value_type!s}." f"{cast(ReverseConnectionEntity, prop_.connection).property_}"
             if source_id not in properties_by_ids:
+                print(f"source_id: {source_id}, first issue")
                 self.issue_list.append(
                     ReversedConnectionNotFeasibleError(
                         id_,
                         "reversed connection",
-                        prop_.property_,
+                        prop_.view_property,
                         str(prop_.view),
                         str(prop_.value_type),
                         cast(ReverseConnectionEntity, prop_.connection).property_,
@@ -366,11 +252,12 @@ class DMSPostValidation:
                 )
 
             elif source_id in properties_by_ids and properties_by_ids[source_id].value_type != prop_.view:
+                print(f"source_id: {source_id}, second issue")
                 self.issue_list.append(
                     ReversedConnectionNotFeasibleError(
                         id_,
                         "view property",
-                        prop_.property_,
+                        prop_.view_property,
                         str(prop_.view),
                         str(prop_.value_type),
                         cast(ReverseConnectionEntity, prop_.connection).property_,

cognite/neat/_rules/models/information/_rules.py

@@ -1,7 +1,6 @@
 import math
 import sys
 from collections.abc import Hashable
-from datetime import datetime
 from typing import TYPE_CHECKING, Any, ClassVar
 
 import pandas as pd
@@ -9,20 +8,16 @@ from pydantic import Field, field_serializer, field_validator, model_validator
 from pydantic_core.core_schema import SerializationInfo
 from rdflib import Namespace, URIRef
 
-from cognite.neat._constants import DEFAULT_NAMESPACE, get_default_prefixes
+from cognite.neat._constants import get_default_prefixes
 from cognite.neat._issues.errors import NeatValueError, PropertyDefinitionError
 from cognite.neat._rules._constants import EntityTypes
 from cognite.neat._rules.models._base_rules import (
     BaseMetadata,
     BaseRules,
     ClassRef,
-    DataModelType,
-    ExtensionCategory,
-    ExtensionCategoryType,
-    MatchType,
+    DataModelAspect,
     PropertyRef,
     RoleTypes,
-    SchemaCompleteness,
     SheetList,
     SheetRow,
 )
@@ -35,10 +30,6 @@ from cognite.neat._rules.models._types import (
     ClassEntityType,
     InformationPropertyType,
     MultiValueTypeType,
-    NamespaceType,
-    PrefixType,
-    StrListType,
-    VersionType,
 )
 from cognite.neat._rules.models.data_types import DataType
 from cognite.neat._rules.models.entities import (
@@ -46,10 +37,8 @@ from cognite.neat._rules.models.entities import (
     ClassEntityList,
     Entity,
     MultiValueTypeInfo,
-    ReferenceEntity,
     Undefined,
     UnknownEntity,
-    URLEntity,
 )
 
 if TYPE_CHECKING:
@@ -64,62 +53,11 @@ else:
 
 class InformationMetadata(BaseMetadata):
     role: ClassVar[RoleTypes] = RoleTypes.information
-    data_model_type: DataModelType = Field(DataModelType.enterprise, alias="dataModelType")
-    schema_: SchemaCompleteness = Field(SchemaCompleteness.partial, alias="schema")
-    extension: ExtensionCategoryType | None = ExtensionCategory.addition
-
-    prefix: PrefixType
-    namespace: NamespaceType
-
-    name: str = Field(
-        alias="title",
-        description="Human readable name of the data model",
-        min_length=1,
-        max_length=255,
-    )
-    description: str | None = Field(None, min_length=1, max_length=1024)
-    version: VersionType
-
-    created: datetime = Field(
-        description=("Date of the data model creation"),
-    )
-
-    updated: datetime = Field(
-        description=("Date of the data model update"),
-    )
-    creator: StrListType = Field(
-        description=(
-            "List of contributors to the data model creation, "
-            "typically information architects are considered as contributors."
-        ),
-    )
-    license: str | None = None
-    rights: str | None = None
-
-    @model_validator(mode="after")
-    def extension_none_but_schema_extend(self) -> Self:
-        if self.extension is None:
-            self.extension = ExtensionCategory.addition
-            return self
-        return self
-
-    @field_validator("schema_", mode="plain")
-    def as_enum_schema(cls, value: str) -> SchemaCompleteness:
-        return SchemaCompleteness(value.strip())
+    aspect: ClassVar[DataModelAspect] = DataModelAspect.logical
 
-    @field_validator("extension", mode="plain")
-    def as_enum_extension(cls, value: str) -> ExtensionCategory:
-        return ExtensionCategory(value.strip())
-
-    @field_validator("data_model_type", mode="plain")
-    def as_enum_model_type(cls, value: str) -> DataModelType:
-        return DataModelType(value.strip())
-
-    def as_identifier(self) -> str:
-        return f"{self.prefix}:{self.name}"
-
-    def get_prefix(self) -> str:
-        return self.prefix
+    # Linking to Conceptual and Physical data model aspects
+    physical: URIRef | None = Field(None, description="Link to the logical data model aspect")
+    conceptual: URIRef | None = Field(None, description="Link to the logical data model aspect")
 
 
 def _get_metadata(context: Any) -> InformationMetadata | None:
@@ -135,42 +73,33 @@ class InformationClass(SheetRow):
     Args:
         class_: The class ID of the class.
         description: A description of the class.
-        parent: The parent class of the class.
-        reference: Reference of the source of the information for given resource
-        match_type: The match type of the resource being described and the source entity.
+        implements: Which classes the current class implements.
     """
 
     class_: ClassEntityType = Field(alias="Class")
     name: str | None = Field(alias="Name", default=None)
     description: str | None = Field(alias="Description", default=None)
-    parent: ClassEntityList | None = Field(alias="Parent Class", default=None)
-    reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
-    match_type: MatchType | None = Field(alias="Match Type", default=None)
-    comment: str | None = Field(alias="Comment", default=None)
+    implements: ClassEntityList | None = Field(alias="Implements", default=None)
 
     def _identifier(self) -> tuple[Hashable, ...]:
         return (self.class_,)
 
-    @field_serializer("reference", when_used="always")
-    def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
-        if isinstance(info.context, dict) and info.context.get("as_reference") is True:
-            return self.class_.dump()
-        return str(value) if value is not None else None
-
     @field_serializer("class_", when_used="unless-none")
     def remove_default_prefix(self, value: Any, info: SerializationInfo) -> str:
         if (metadata := _get_metadata(info.context)) and isinstance(value, Entity):
             return value.dump(prefix=metadata.prefix, version=metadata.version)
         return str(value)
 
-    @field_serializer("parent", when_used="unless-none")
+    @field_serializer("implements", when_used="unless-none")
     def remove_default_prefixes(self, value: Any, info: SerializationInfo) -> str:
         if isinstance(value, list) and (metadata := _get_metadata(info.context)):
             return ",".join(
-                parent.dump(prefix=metadata.prefix, version=metadata.version)
-                if isinstance(parent, Entity)
-                else str(parent)
-                for parent in value
+                (
+                    class_.dump(prefix=metadata.prefix, version=metadata.version)
+                    if isinstance(class_, Entity)
+                    else str(class_)
+                )
+                for class_ in value
             )
         return ",".join(str(value) for value in value)
 
@@ -191,8 +120,6 @@ class InformationProperty(SheetRow):
         min_count: Minimum count of the property values. Defaults to 0
         max_count: Maximum count of the property values. Defaults to None
         default: Default value of the property
-        reference: Reference to the source of the information, HTTP URI
-        match_type: The match type of the resource being described and the source entity.
         transformation: Actual rule for the transformation from source to target representation of
               knowledge graph. Defaults to None (no transformation)
     """
@@ -207,10 +134,7 @@ class InformationProperty(SheetRow):
     min_count: int | None = Field(alias="Min Count", default=None)
     max_count: int | float | None = Field(alias="Max Count", default=None)
     default: Any | None = Field(alias="Default", default=None)
-    reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
-    match_type: MatchType | None = Field(alias="Match Type", default=None)
     transformation: RDFPath | None = Field(alias="Transformation", default=None)
-    comment: str | None = Field(alias="Comment", default=None)
     inherited: bool = Field(
         default=False,
         exclude=True,
@@ -281,19 +205,6 @@ class InformationProperty(SheetRow):
             return None
         return value
 
-    @field_serializer("reference", when_used="always")
-    def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
-        # When rules as dumped as reference, we set the reference to the class
-        if isinstance(info.context, dict) and info.context.get("as_reference") is True:
-            return str(
-                ReferenceEntity(
-                    prefix=str(self.class_.prefix),
-                    suffix=self.class_.suffix,
-                    property=self.property_,
-                )
-            )
-        return str(value) if value is not None else None
-
     @field_serializer("class_", "value_type", when_used="unless-none")
     def remove_default_prefix(self, value: Any, info: SerializationInfo) -> str:
         if (metadata := _get_metadata(info.context)) and isinstance(value, Entity):
@@ -331,8 +242,6 @@ class InformationRules(BaseRules):
     properties: SheetList[InformationProperty] = Field(alias="Properties")
     classes: SheetList[InformationClass] = Field(alias="Classes")
     prefixes: dict[str, Namespace] = Field(default_factory=get_default_prefixes, alias="Prefixes")
-    last: "InformationRules | None" = Field(None, alias="Last")
-    reference: "InformationRules | None" = Field(None, alias="Reference")
 
     @field_validator("prefixes", mode="before")
     def parse_str(cls, values: Any) -> Any:
@@ -355,10 +264,10 @@ class InformationRules(BaseRules):
             if property_.class_.prefix is Undefined:
                 property_.class_.prefix = self.metadata.prefix
 
-        # update parent classes
+        # update implements
         for class_ in self.classes:
-            if class_.parent:
-                for parent in class_.parent:
+            if class_.implements:
+                for parent in class_.implements:
                     if not isinstance(parent.prefix, str):
                         parent.prefix = self.metadata.prefix
             if class_.class_.prefix is Undefined:
@@ -387,14 +296,10 @@ class InformationRules(BaseRules):
             "type": "Logical Data Model",
             "intended for": "Information Architect",
             "name": self.metadata.name,
-            "external_id": self.metadata.prefix,
+            "external_id": self.metadata.external_id,
             "version": self.metadata.version,
             "classes": len(self.classes),
             "properties": len(self.properties),
         }
 
         return pd.DataFrame([summary]).T.rename(columns={0: ""})._repr_html_()  # type: ignore
-
-    @property
-    def id_(self) -> URIRef:
-        return DEFAULT_NAMESPACE[f"data-model/verified/info/{self.metadata.prefix}/{self.metadata.version}"]

cognite/neat/_rules/models/information/_rules_input.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import Any, Literal
+from typing import Any
 
 import pandas as pd
 from rdflib import Namespace, URIRef
@@ -25,19 +25,16 @@ from ._rules import (
 
 @dataclass
 class InformationInputMetadata(InputComponent[InformationMetadata]):
-    schema_: Literal["complete", "partial", "extended"]
-    prefix: str
-    namespace: str
+    space: str
+    external_id: str
     version: str
     creator: str
-    data_model_type: Literal["solution", "enterprise"] = "enterprise"
-    extension: Literal["addition", "reshape", "rebuild"] = "addition"
     name: str | None = None
     description: str | None = None
     created: datetime | str | None = None
     updated: datetime | str | None = None
-    license: str | None = None
-    rights: str | None = None
+    physical: str | None = None
+    conceptual: str | None = None
 
     @classmethod
     def _get_verified_cls(cls) -> type[InformationMetadata]:
@@ -51,6 +48,25 @@ class InformationInputMetadata(InputComponent[InformationMetadata]):
             output["updated"] = datetime.now()
         return output
 
+    @property
+    def prefix(self) -> str:
+        return self.space
+
+    @property
+    def identifier(self) -> URIRef:
+        """Globally unique identifier for the data model.
+
+        !!! note
+            Unlike namespace, the identifier does not end with "/" or "#".
+
+        """
+        return DEFAULT_NAMESPACE[f"data-model/unverified/logical/{self.space}/{self.external_id}/{self.version}"]
+
+    @property
+    def namespace(self) -> Namespace:
+        """Namespace for the data model used for the entities in the data model."""
+        return Namespace(f"{self.identifier}/")
+
 
 @dataclass
 class InformationInputProperty(InputComponent[InformationProperty]):
@@ -59,12 +75,9 @@ class InformationInputProperty(InputComponent[InformationProperty]):
     value_type: DataType | ClassEntity | MultiValueTypeInfo | UnknownEntity | str
     name: str | None = None
     description: str | None = None
-    comment: str | None = None
     min_count: int | None = None
     max_count: int | float | None = None
     default: Any | None = None
-    reference: str | None = None
-    match_type: str | None = None
     transformation: str | None = None
     # Only used internally
     inherited: bool = False
@@ -85,10 +98,7 @@ class InformationInputClass(InputComponent[InformationClass]):
     class_: ClassEntity | str
     name: str | None = None
     description: str | None = None
-    comment: str | None = None
-    parent: str | list[ClassEntity] | None = None
-    reference: str | None = None
-    match_type: str | None = None
+    implements: str | list[ClassEntity] | None = None
 
     @classmethod
     def _get_verified_cls(cls) -> type[InformationClass]:
@@ -101,12 +111,12 @@ class InformationInputClass(InputComponent[InformationClass]):
     def dump(self, default_prefix: str, **kwargs) -> dict[str, Any]:  # type: ignore[override]
         output = super().dump()
         parent: list[ClassEntity] | None = None
-        if isinstance(self.parent, str):
-            parent = [ClassEntity.load(parent, prefix=default_prefix) for parent in self.parent.split(",")]
-        elif isinstance(self.parent, list):
-            parent = [ClassEntity.load(parent_, prefix=default_prefix) for parent_ in self.parent]
+        if isinstance(self.implements, str):
+            parent = [ClassEntity.load(parent, prefix=default_prefix) for parent in self.implements.split(",")]
+        elif isinstance(self.implements, list):
+            parent = [ClassEntity.load(parent_, prefix=default_prefix) for parent_ in self.implements]
         output["Class"] = ClassEntity.load(self.class_, prefix=default_prefix)
-        output["Parent Class"] = parent
+        output["Implements"] = parent
         return output
 
 
@@ -116,8 +126,6 @@ class InformationInputRules(InputRules[InformationRules]):
     properties: list[InformationInputProperty] = field(default_factory=list)
     classes: list[InformationInputClass] = field(default_factory=list)
     prefixes: dict[str, Namespace] | None = None
-    last: "InformationInputRules | None" = None
-    reference: "InformationInputRules | None" = None
 
     @classmethod
     def _get_verified_cls(cls) -> type[InformationRules]:
@@ -125,26 +133,12 @@ class InformationInputRules(InputRules[InformationRules]):
 
     def dump(self) -> dict[str, Any]:
         default_prefix = self.metadata.prefix
-        reference: dict[str, Any] | None = None
-        if isinstance(self.reference, InformationInputRules):
-            reference = self.reference.dump()
-        elif isinstance(self.reference, InformationRules):
-            # We need to load through the InformationRulesInput to set the correct default space and version
-            reference = InformationInputRules.load(self.reference.model_dump()).dump()
-        last: dict[str, Any] | None = None
-        if isinstance(self.last, InformationInputRules):
-            last = self.last.dump()
-        elif isinstance(self.last, InformationRules):
-            # We need to load through the InformationRulesInput to set the correct default space and version
-            last = InformationInputRules.load(self.last.model_dump()).dump()
 
         return dict(
             Metadata=self.metadata.dump(),
             Properties=[prop.dump(default_prefix) for prop in self.properties],
             Classes=[class_.dump(default_prefix) for class_ in self.classes],
             Prefixes=self.prefixes,
-            Last=last,
-            Reference=reference,
         )
 
     def _repr_html_(self) -> str:
@@ -152,14 +146,11 @@ class InformationInputRules(InputRules[InformationRules]):
             "type": "Logical Data Model",
             "intended for": "Information Architect",
             "name": self.metadata.name,
-            "external_id": self.metadata.prefix,
+            "external_id": self.metadata.external_id,
+            "space": self.metadata.space,
             "version": self.metadata.version,
             "classes": len(self.classes),
             "properties": len(self.properties),
         }
 
         return pd.DataFrame([summary]).T.rename(columns={0: ""})._repr_html_()  # type: ignore
-
-    @property
-    def id_(self) -> URIRef:
-        return DEFAULT_NAMESPACE[f"data-model/unverified/info/{self.metadata.prefix}/{self.metadata.version}"]