cognite-neat 0.77.2__py3-none-any.whl → 0.77.4__py3-none-any.whl

cognite/neat/rules/models/dms/_converter.py

@@ -1,6 +1,5 @@
 import warnings
-from datetime import datetime
-from typing import cast
+from typing import TYPE_CHECKING, cast
 
 from rdflib import Namespace
 
@@ -20,7 +19,10 @@ from cognite.neat.rules.models.entities import (
 )
 from cognite.neat.rules.models.information._rules import InformationRules
 
-from ._rules import DMSProperty, DMSRules, DMSView
+from ._rules import DMSMetadata, DMSProperty, DMSRules, DMSView
+
+if TYPE_CHECKING:
+    from cognite.neat.rules.models.information._rules import InformationMetadata
 
 
 class _DMSRulesConverter:
@@ -32,31 +34,16 @@ class _DMSRulesConverter:
 
     def as_information_architect_rules(
         self,
-        created: datetime | None = None,
-        updated: datetime | None = None,
-        name: str | None = None,
-        namespace: Namespace | None = None,
     ) -> "InformationRules":
         from cognite.neat.rules.models.information._rules import (
             InformationClass,
-            InformationMetadata,
             InformationProperty,
             InformationRules,
         )
 
         dms = self.dms.metadata
-        prefix = dms.space
 
-        metadata = InformationMetadata(
-            schema_=dms.schema_,
-            prefix=prefix,
-            namespace=namespace or Namespace(f"https://purl.orgl/neat/{prefix}/"),
-            version=dms.version,
-            name=name or dms.name or "Missing name",
-            creator=dms.creator,
-            created=dms.created or created or datetime.now(),
-            updated=dms.updated or updated or datetime.now(),
-        )
+        metadata = self._convert_metadata_to_info(dms)
 
         classes = [
             InformationClass(
@@ -111,6 +98,25 @@ class _DMSRulesConverter:
             reference=self.dms.reference.as_information_architect_rules() if self.dms.reference else None,
         )
 
+    @classmethod
+    def _convert_metadata_to_info(cls, metadata: DMSMetadata) -> "InformationMetadata":
+        from cognite.neat.rules.models.information._rules import InformationMetadata
+
+        prefix = metadata.space
+        return InformationMetadata(
+            schema_=metadata.schema_,
+            data_model_type=metadata.data_model_type,
+            extension=metadata.extension,
+            prefix=prefix,
+            namespace=Namespace(f"https://purl.orgl/neat/{prefix}/"),
+            version=metadata.version,
+            description=metadata.description,
+            name=metadata.name or metadata.external_id,
+            creator=metadata.creator,
+            created=metadata.created,
+            updated=metadata.updated,
+        )
+
     @classmethod
     def _get_class_reference(cls, view: DMSView) -> ReferenceEntity | None:
         parents_other_namespace = [parent for parent in view.implements or [] if parent.prefix != view.class_.prefix]

cognite/neat/rules/models/dms/_exporter.py

@@ -86,7 +86,8 @@ class _DMSExporter:
 
         if self._ref_schema:
             output.reference = self._ref_schema
-
+        if self.rules.last:
+            output.last = self.rules.last.as_schema()
         return output
 
     def _create_spaces(

cognite/neat/rules/models/dms/_rules.py

@@ -58,7 +58,7 @@ _DEFAULT_VERSION = "1"
 
 class DMSMetadata(BaseMetadata):
     role: ClassVar[RoleTypes] = RoleTypes.dms_architect
-    data_model_type: DataModelType = Field(DataModelType.solution, alias="dataModelType")
+    data_model_type: DataModelType = Field(DataModelType.enterprise, alias="dataModelType")
     schema_: SchemaCompleteness = Field(alias="schema")
     extension: ExtensionCategory = ExtensionCategory.addition
     space: ExternalIdType
@@ -146,10 +146,11 @@ class DMSMetadata(BaseMetadata):
         return description, creator
 
     @classmethod
-    def from_data_model(cls, data_model: dm.DataModelApply) -> "DMSMetadata":
+    def from_data_model(cls, data_model: dm.DataModelApply, has_reference: bool) -> "DMSMetadata":
         description, creator = cls._get_description_and_creator(data_model.description)
         return cls(
             schema_=SchemaCompleteness.complete,
+            data_model_type=DataModelType.solution if has_reference else DataModelType.enterprise,
             space=data_model.space,
             name=data_model.name or None,
             description=description,

cognite/neat/rules/models/dms/_rules_input.py

@@ -3,9 +3,7 @@ from dataclasses import dataclass
 from datetime import datetime
 from typing import Any, Literal, cast, overload
 
-from pydantic import BaseModel
-
-from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
+from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness, _add_alias
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import (
     ClassEntity,
@@ -44,8 +42,9 @@ class DMSMetadataInput:
             external_id=data.get("external_id"),  # type: ignore[arg-type]
             creator=data.get("creator"),  # type: ignore[arg-type]
             version=data.get("version"),  # type: ignore[arg-type]
-            extension=data.get("extension", "addition"),
-            data_model_type=data.get("data_model_type", "solution"),
+            # safeguard from empty cell, i.e. if key provided by value None
+            extension=data.get("extension", "addition") or "addition",
+            data_model_type=data.get("data_model_type", "solution") or "solution",
             name=data.get("name"),
             description=data.get("description"),
             created=data.get("created"),
@@ -355,9 +354,3 @@ class DMSRulesInput:
             Last=last,
             Reference=reference,
         )
-
-
-def _add_alias(data: dict[str, Any], base_model: type[BaseModel]) -> None:
-    for field_name, field_ in base_model.model_fields.items():
-        if field_name not in data and field_.alias in data:
-            data[field_name] = data[field_.alias]

cognite/neat/rules/models/dms/_schema.py

@@ -12,6 +12,7 @@ from cognite.client import CogniteClient
 from cognite.client import data_modeling as dm
 from cognite.client.data_classes import DatabaseWrite, DatabaseWriteList, TransformationWrite, TransformationWriteList
 from cognite.client.data_classes.data_modeling import ViewApply
+from cognite.client.data_classes.data_modeling.views import ReverseDirectRelation
 from cognite.client.data_classes.transformations.common import Edges, EdgeType, Nodes, ViewInfo
 
 from cognite.neat.rules import issues
@@ -28,6 +29,7 @@ from cognite.neat.rules.issues.dms import (
     MissingSourceViewError,
     MissingSpaceError,
     MissingViewError,
+    MissingViewInModelWarning,
 )
 from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE
 from cognite.neat.utils.cdf_classes import (
@@ -146,8 +148,26 @@ class DMSSchema:
         space_write = space_read.as_write()
 
         view_loader = ViewLoader(client)
-        # We need to include parent views in the schema to make sure that the schema is valid.
+
         existing_view_ids = set(views.as_ids())
+
+        # We need to include all views the edges/direct relations are pointing to have a complete schema.
+        connection_referenced_view_ids: set[dm.ViewId] = set()
+        for view in views:
+            connection_referenced_view_ids |= cls._connection_references(view)
+        connection_referenced_view_ids = connection_referenced_view_ids - existing_view_ids
+        if connection_referenced_view_ids:
+            warnings.warn(
+                MissingViewInModelWarning(data_model.as_id(), connection_referenced_view_ids), UserWarning, stacklevel=2
+            )
+            connection_referenced_views = view_loader.retrieve(list(connection_referenced_view_ids))
+            if failed := connection_referenced_view_ids - set(connection_referenced_views.as_ids()):
+                warnings.warn(
+                    issues.importing.FailedImportWarning({repr(v) for v in failed}), UserWarning, stacklevel=2
+                )
+            views.extend(connection_referenced_views)
+
+        # We need to include parent views in the schema to make sure that the schema is valid.
         parent_view_ids = {parent for view in views for parent in view.implements or []}
         parents = view_loader.retrieve_all_parents(list(parent_view_ids - existing_view_ids))
         views.extend([parent for parent in parents if parent.as_id() not in existing_view_ids])
@@ -203,6 +223,21 @@ class DMSSchema:
             reference=ref_schema,
         )
 
+    @classmethod
+    def _connection_references(cls, view: dm.View) -> set[dm.ViewId]:
+        view_ids: set[dm.ViewId] = set()
+        for prop in (view.properties or {}).values():
+            if isinstance(prop, dm.MappedProperty) and isinstance(prop.type, dm.DirectRelation):
+                if prop.source:
+                    view_ids.add(prop.source)
+            elif isinstance(prop, dm.EdgeConnection):
+                view_ids.add(prop.source)
+                if prop.edge_source:
+                    view_ids.add(prop.edge_source)
+            elif isinstance(prop, ReverseDirectRelation):
+                view_ids.add(prop.source)
+        return view_ids
+
     @classmethod
     def from_directory(cls, directory: str | Path) -> Self:
         """Load a schema from a directory containing YAML files.
@@ -481,10 +516,11 @@ class DMSSchema:
         defined_spaces = self.spaces.copy()
         defined_containers = self.containers.copy()
         defined_views = self.views.copy()
-        if self.reference:
-            defined_spaces |= self.reference.spaces
-            defined_containers |= self.reference.containers
-            defined_views |= self.reference.views
+        for other_schema in [self.reference, self.last]:
+            if other_schema:
+                defined_spaces |= other_schema.spaces
+                defined_containers |= other_schema.containers
+                defined_views |= other_schema.views
 
         for container in self.containers.values():
             if container.space not in defined_spaces:

cognite/neat/rules/models/dms/_serializer.py

@@ -37,7 +37,7 @@ class _DMSRulesSerializer:
                 DMSProperty.model_fields[field].alias or field for field in self.properties_fields
             ]
             self.views_fields = [DMSView.model_fields[field].alias or field for field in self.views_fields]
-            self.container_fields = [
+            self.containers_fields = [
                 DMSContainer.model_fields[field].alias or field for field in self.containers_fields
             ]
             self.prop_view = DMSProperty.model_fields[self.prop_view].alias or self.prop_view

cognite/neat/rules/models/dms/_validation.py

@@ -5,12 +5,13 @@ from cognite.client import data_modeling as dm
 
 from cognite.neat.rules import issues
 from cognite.neat.rules.issues import IssueList
-from cognite.neat.rules.models._base import ExtensionCategory, SchemaCompleteness
+from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import ContainerEntity
 from cognite.neat.rules.models.wrapped_entities import RawFilter
 
 from ._rules import DMSProperty, DMSRules
+from ._schema import DMSSchema
 
 
 class DMSPostValidation:
@@ -26,12 +27,17 @@ class DMSPostValidation:
         self.issue_list = IssueList()
 
     def validate(self) -> IssueList:
-        self._validate_best_practices()
+        self._validate_raw_filter()
         self._consistent_container_properties()
+
         self._referenced_views_and_containers_are_existing()
-        self._validate_extension()
-        self._validate_schema()
-        self._validate_performance()
+        if self.metadata.schema_ is SchemaCompleteness.extended:
+            self._validate_extension()
+        if self.metadata.schema_ is SchemaCompleteness.partial:
+            return self.issue_list
+        dms_schema = self.rules.as_schema()
+        self.issue_list.extend(dms_schema.validate())
+        self._validate_performance(dms_schema)
         return self.issue_list
 
     def _consistent_container_properties(self) -> None:
@@ -106,8 +112,9 @@ class DMSPostValidation:
         self.issue_list.extend(errors)
 
     def _referenced_views_and_containers_are_existing(self) -> None:
-        # There two checks are done in the same method to raise all the errors at once.
         defined_views = {view.view.as_id() for view in self.views}
+        if self.metadata.schema_ is SchemaCompleteness.extended and self.rules.last:
+            defined_views |= {view.view.as_id() for view in self.rules.last.views}
 
         errors: list[issues.NeatValidationError] = []
         for prop_no, prop in enumerate(self.properties):
@@ -125,6 +132,11 @@ class DMSPostValidation:
                 )
         if self.metadata.schema_ is SchemaCompleteness.complete:
             defined_containers = {container.container.as_id() for container in self.containers or []}
+            if self.metadata.data_model_type == DataModelType.solution and self.rules.reference:
+                defined_containers |= {
+                    container.container.as_id() for container in self.rules.reference.containers or []
+                }
+
             for prop_no, prop in enumerate(self.properties):
                 if prop.container and (container_id := prop.container.as_id()) not in defined_containers:
                     errors.append(
@@ -157,19 +169,16 @@ class DMSPostValidation:
     def _validate_extension(self) -> None:
         if self.metadata.schema_ is not SchemaCompleteness.extended:
             return None
-        if not self.rules.reference:
-            raise ValueError("The schema is set to 'extended', but no reference rules are provided to validate against")
-        is_solution = self.metadata.space != self.rules.reference.metadata.space
-        if is_solution:
-            return None
+        if not self.rules.last:
+            raise ValueError("The schema is set to 'extended', but no last rules are provided to validate against")
         if self.metadata.extension is ExtensionCategory.rebuild:
             # Everything is allowed
             return None
-        # Is an extension of an existing model.
         user_schema = self.rules.as_schema()
-        ref_schema = self.rules.reference.as_schema()
         new_containers = user_schema.containers.copy()
-        existing_containers = ref_schema.containers.copy()
+
+        last_schema = self.rules.last.as_schema()
+        existing_containers = last_schema.containers.copy()
 
         for container_id, container in new_containers.items():
             existing_container = existing_containers.get(container_id)
@@ -189,14 +198,12 @@ class DMSPostValidation:
                 )
             )
 
-        if self.metadata.extension is ExtensionCategory.reshape and self.issue_list:
-            return None
-        elif self.metadata.extension is ExtensionCategory.reshape:
+        if self.metadata.extension is ExtensionCategory.reshape:
             # Reshape allows changes to views
             return None
 
         new_views = user_schema.views.copy()
-        existing_views = ref_schema.views.copy()
+        existing_views = last_schema.views.copy()
         for view_id, view in new_views.items():
             existing_view = existing_views.get(view_id)
             if not existing_view or existing_view == view:
@@ -213,14 +220,7 @@ class DMSPostValidation:
                 )
             )
 
-    def _validate_performance(self) -> None:
-        # we can only validate performance on complete schemas due to the need
-        # to access all the container mappings
-        if self.metadata.schema_ is not SchemaCompleteness.complete:
-            return None
-
-        dms_schema = self.rules.as_schema()
-
+    def _validate_performance(self, dms_schema: DMSSchema) -> None:
         for view_id, view in dms_schema.views.items():
             mapped_containers = dms_schema._get_mapped_container_from_view(view_id)
 
@@ -243,7 +243,7 @@ class DMSPostValidation:
                         )
                     )
 
-    def _validate_best_practices(self) -> None:
+    def _validate_raw_filter(self) -> None:
         for view in self.views:
             if view.filter_ and isinstance(view.filter_, RawFilter):
                 self.issue_list.append(
@@ -264,36 +264,3 @@ class DMSPostValidation:
         existing_properties = existing_dumped.get("properties", {})
         changed_properties = [prop for prop in new_properties if new_properties[prop] != existing_properties.get(prop)]
         return changed_attributes, changed_properties
-
-    def _validate_schema(self) -> None:
-        if self.metadata.schema_ is SchemaCompleteness.partial:
-            return None
-        elif self.metadata.schema_ is SchemaCompleteness.complete:
-            rules: DMSRules = self.rules
-        elif self.metadata.schema_ is SchemaCompleteness.extended:
-            if not self.rules.reference:
-                raise ValueError(
-                    "The schema is set to 'extended', but no reference rules are provided to validate against"
-                )
-            # This is an extension of the reference rules, we need to merge the two
-            rules = self.rules.model_copy(deep=True)
-            rules.properties.extend(self.rules.reference.properties.data)
-            existing_views = {view.view.as_id() for view in rules.views}
-            rules.views.extend([view for view in self.rules.reference.views if view.view.as_id() not in existing_views])
-            if rules.containers and self.rules.reference.containers:
-                existing_containers = {container.container.as_id() for container in rules.containers.data}
-                rules.containers.extend(
-                    [
-                        container
-                        for container in self.rules.reference.containers
-                        if container.container.as_id() not in existing_containers
-                    ]
-                )
-            elif not rules.containers and self.rules.reference.containers:
-                rules.containers = self.rules.reference.containers
-        else:
-            raise ValueError("Unknown schema completeness")
-
-        schema = rules.as_schema()
-        errors = schema.validate()
-        self.issue_list.extend(errors)

cognite/neat/rules/models/information/__init__.py

@@ -1,3 +1,10 @@
 from ._rules import InformationClass, InformationMetadata, InformationProperty, InformationRules
+from ._rules_input import InformationRulesInput
 
-__all__ = ["InformationRules", "InformationMetadata", "InformationClass", "InformationProperty"]
+__all__ = [
+    "InformationRules",
+    "InformationMetadata",
+    "InformationClass",
+    "InformationProperty",
+    "InformationRulesInput",
+]

cognite/neat/rules/models/information/_converter.py

@@ -1,14 +1,11 @@
 import re
 from collections import defaultdict
-from datetime import datetime
-from typing import Literal
+from typing import TYPE_CHECKING, Literal
 
 from cognite.neat.rules.models._base import (
-    DataModelType,
     SheetList,
 )
 from cognite.neat.rules.models.data_types import DataType
-from cognite.neat.rules.models.dms._rules import DMSProperty, DMSRules, DMSView
 from cognite.neat.rules.models.domain import DomainRules
 from cognite.neat.rules.models.entities import (
     ClassEntity,
@@ -22,6 +19,9 @@ from cognite.neat.rules.models.entities import (
 
 from ._rules import InformationClass, InformationMetadata, InformationProperty, InformationRules
 
+if TYPE_CHECKING:
+    from cognite.neat.rules.models.dms._rules import DMSMetadata, DMSProperty, DMSRules
+
 
 class _InformationRulesConverter:
     def __init__(self, information: InformationRules):
@@ -30,30 +30,18 @@ class _InformationRulesConverter:
     def as_domain_rules(self) -> DomainRules:
         raise NotImplementedError("DomainRules not implemented yet")
 
-    def as_dms_architect_rules(self, created: datetime | None = None, updated: datetime | None = None) -> "DMSRules":
+    def as_dms_architect_rules(self) -> "DMSRules":
         from cognite.neat.rules.models.dms._rules import (
             DMSContainer,
-            DMSMetadata,
             DMSProperty,
             DMSRules,
+            DMSView,
         )
 
         info_metadata = self.information.metadata
         default_version = info_metadata.version
         default_space = self._to_space(info_metadata.prefix)
-        space = self._to_space(info_metadata.prefix)
-
-        metadata = DMSMetadata(
-            schema_=info_metadata.schema_,
-            space=space,
-            data_model_type=DataModelType.solution if self.information.reference else DataModelType.enterprise,
-            version=info_metadata.version,
-            external_id=info_metadata.name.replace(" ", "_").lower(),
-            creator=info_metadata.creator,
-            name=info_metadata.name,
-            created=created or datetime.now(),
-            updated=updated or datetime.now(),
-        )
+        metadata = self._convert_metadata_to_dms(info_metadata)
 
         properties_by_class: dict[str, list[DMSProperty]] = defaultdict(list)
         for prop in self.information.properties:
@@ -111,6 +99,26 @@ class _InformationRulesConverter:
             reference=self.information.reference.as_dms_architect_rules() if self.information.reference else None,
         )
 
+    @classmethod
+    def _convert_metadata_to_dms(cls, metadata: InformationMetadata) -> "DMSMetadata":
+        from cognite.neat.rules.models.dms._rules import (
+            DMSMetadata,
+        )
+
+        space = cls._to_space(metadata.prefix)
+
+        return DMSMetadata(
+            schema_=metadata.schema_,
+            space=space,
+            data_model_type=metadata.data_model_type,
+            version=metadata.version,
+            external_id=metadata.name.replace(" ", "_").lower(),
+            creator=metadata.creator,
+            name=metadata.name,
+            created=metadata.created,
+            updated=metadata.updated,
+        )
+
     @classmethod
     def _as_dms_property(cls, prop: InformationProperty, default_space: str, default_version: str) -> "DMSProperty":
         """This creates the first"""