cognite-neat 0.96.6__py3-none-any.whl → 0.97.0__py3-none-any.whl

cognite/neat/_rules/models/dms/_validation.py

@@ -1,18 +1,20 @@
 from collections import defaultdict
-from typing import Any, ClassVar
+from typing import Any, ClassVar, cast
 
 from cognite.client import data_modeling as dm
 
-from cognite.neat._constants import DMS_CONTAINER_PROPERTY_SIZE_LIMIT
+from cognite.neat._constants import COGNITE_MODELS, DMS_CONTAINER_PROPERTY_SIZE_LIMIT
 from cognite.neat._issues import IssueList, NeatError, NeatIssue, NeatIssueList
 from cognite.neat._issues.errors import (
     PropertyDefinitionDuplicatedError,
     ResourceChangedError,
     ResourceNotDefinedError,
 )
+from cognite.neat._issues.errors._properties import ReversedConnectionNotFeasibleError
 from cognite.neat._issues.warnings import (
     NotSupportedHasDataFilterLimitWarning,
     NotSupportedViewContainerLimitWarning,
+    UndefinedViewWarning,
 )
 from cognite.neat._issues.warnings.user_modeling import (
     NotNeatSupportedFilterWarning,
@@ -21,6 +23,10 @@ from cognite.neat._issues.warnings.user_modeling import (
 from cognite.neat._rules.models._base_rules import DataModelType, ExtensionCategory, SchemaCompleteness
 from cognite.neat._rules.models.data_types import DataType
 from cognite.neat._rules.models.entities import ContainerEntity, RawFilter
+from cognite.neat._rules.models.entities._single_value import (
+    ReverseConnectionEntity,
+    ViewEntity,
+)
 
 from ._rules import DMSProperty, DMSRules
 from ._schema import DMSSchema
@@ -45,6 +51,8 @@ class DMSPostValidation:
     def validate(self) -> NeatIssueList:
         self._validate_raw_filter()
         self._consistent_container_properties()
+        self._validate_value_type_existence()
+        self._validate_reverse_connections()
 
         self._referenced_views_and_containers_are_existing_and_proper_size()
         if self.metadata.schema_ is SchemaCompleteness.extended:
@@ -318,6 +326,60 @@ class DMSPostValidation:
                     NotNeatSupportedFilterWarning(view.view.as_id()),
                 )
 
+    def _validate_value_type_existence(self) -> None:
+        views = {prop_.view for prop_ in self.properties}.union({view_.view for view_ in self.views})
+
+        for prop_ in self.properties:
+            if isinstance(prop_.value_type, ViewEntity) and prop_.value_type not in views:
+                self.issue_list.append(
+                    UndefinedViewWarning(
+                        str(prop_.view),
+                        str(prop_.value_type),
+                        prop_.property_,
+                    )
+                )
+
+    def _validate_reverse_connections(self) -> None:
+        # do not check for reverse connections in Cognite models
+        if self.metadata.as_data_model_id() in COGNITE_MODELS:
+            return None
+
+        properties_by_ids = {f"{prop_.view!s}.{prop_.property_}": prop_ for prop_ in self.properties}
+        reversed_by_ids = {
+            id_: prop_
+            for id_, prop_ in properties_by_ids.items()
+            if prop_.connection and isinstance(prop_.connection, ReverseConnectionEntity)
+        }
+
+        for id_, prop_ in reversed_by_ids.items():
+            source_id = f"{prop_.value_type!s}." f"{cast(ReverseConnectionEntity, prop_.connection).property_}"
+            if source_id not in properties_by_ids:
+                self.issue_list.append(
+                    ReversedConnectionNotFeasibleError(
+                        id_,
+                        "reversed connection",
+                        prop_.property_,
+                        str(prop_.view),
+                        str(prop_.value_type),
+                        cast(ReverseConnectionEntity, prop_.connection).property_,
+                    )
+                )
+
+            elif source_id in properties_by_ids and properties_by_ids[source_id].value_type != prop_.view:
+                self.issue_list.append(
+                    ReversedConnectionNotFeasibleError(
+                        id_,
+                        "view property",
+                        prop_.property_,
+                        str(prop_.view),
+                        str(prop_.value_type),
+                        cast(ReverseConnectionEntity, prop_.connection).property_,
+                    )
+                )
+
+            else:
+                continue
+
     @staticmethod
     def _changed_attributes_and_properties(
         new_dumped: dict[str, Any], existing_dumped: dict[str, Any]

cognite/neat/_rules/models/domain.py

@@ -4,7 +4,9 @@ from dataclasses import dataclass, field
 from typing import ClassVar
 
 from pydantic import Field, field_serializer, field_validator
+from rdflib import URIRef
 
+from cognite.neat._constants import DEFAULT_NAMESPACE
 from cognite.neat._rules.models.data_types import DataType
 from cognite.neat._rules.models.entities import ClassEntity, ClassEntityList
 
@@ -76,6 +78,10 @@ class DomainRules(BaseRules):
     last: "DomainRules | None" = Field(None, alias="Last")
     reference: "DomainRules | None" = Field(None, alias="Reference")
 
+    @property
+    def id_(self) -> URIRef:
+        return DEFAULT_NAMESPACE["data-model/verified/domain"]
+
 
 @dataclass
 class DomainInputMetadata(InputComponent[DomainMetadata]):
@@ -124,3 +130,7 @@ class DomainInputRules(InputRules[DomainRules]):
     @classmethod
     def _get_verified_cls(cls) -> type[DomainRules]:
         return DomainRules
+
+    @property
+    def id_(self) -> URIRef:
+        return DEFAULT_NAMESPACE["data-model/unverified/domain"]

cognite/neat/_rules/models/entities/_loaders.py

@@ -63,11 +63,9 @@ def load_connection(
     default_space: str,
     default_version: str,
 ) -> Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None:
-    if (
-        isinstance(raw, EdgeEntity | ReverseConnectionEntity)
-        or raw is None
-        or (isinstance(raw, str) and raw == "direct")
-    ):
+    if isinstance(raw, str) and raw.lower() == "direct":
+        return "direct"  # type: ignore[return-value]
+    elif isinstance(raw, EdgeEntity | ReverseConnectionEntity) or raw is None:
         return raw  # type: ignore[return-value]
     elif isinstance(raw, str) and raw.startswith("edge"):
         return EdgeEntity.load(raw, space=default_space, version=default_version)  # type: ignore[return-value]

cognite/neat/_rules/models/information/_rules.py

@@ -7,9 +7,9 @@ from typing import TYPE_CHECKING, Any, ClassVar
 import pandas as pd
 from pydantic import Field, field_serializer, field_validator, model_validator
 from pydantic_core.core_schema import SerializationInfo
-from rdflib import Namespace
+from rdflib import Namespace, URIRef
 
-from cognite.neat._constants import get_default_prefixes
+from cognite.neat._constants import DEFAULT_NAMESPACE, get_default_prefixes
 from cognite.neat._issues.errors import NeatValueError, PropertyDefinitionError
 from cognite.neat._rules._constants import EntityTypes
 from cognite.neat._rules.models._base_rules import (
@@ -394,3 +394,7 @@ class InformationRules(BaseRules):
         }
 
         return pd.DataFrame([summary]).T.rename(columns={0: ""})._repr_html_()  # type: ignore
+
+    @property
+    def id_(self) -> URIRef:
+        return DEFAULT_NAMESPACE[f"data-model/verified/info/{self.metadata.prefix}/{self.metadata.version}"]

cognite/neat/_rules/models/information/_rules_input.py

@@ -3,8 +3,9 @@ from datetime import datetime
 from typing import Any, Literal
 
 import pandas as pd
-from rdflib import Namespace
+from rdflib import Namespace, URIRef
 
+from cognite.neat._constants import DEFAULT_NAMESPACE
 from cognite.neat._rules.models._base_input import InputComponent, InputRules
 from cognite.neat._rules.models.data_types import DataType
 from cognite.neat._rules.models.entities import (
@@ -158,3 +159,7 @@ class InformationInputRules(InputRules[InformationRules]):
         }
 
         return pd.DataFrame([summary]).T.rename(columns={0: ""})._repr_html_()  # type: ignore
+
+    @property
+    def id_(self) -> URIRef:
+        return DEFAULT_NAMESPACE[f"data-model/unverified/info/{self.metadata.prefix}/{self.metadata.version}"]

cognite/neat/_rules/transformers/_base.py

@@ -2,6 +2,7 @@ from abc import ABC, abstractmethod
 from collections.abc import MutableSequence
 from typing import Generic, TypeVar
 
+from cognite.neat._constants import DEFAULT_NAMESPACE
 from cognite.neat._issues import IssueList, NeatError
 from cognite.neat._issues.errors import NeatTypeError, NeatValueError
 from cognite.neat._rules._shared import (
@@ -12,6 +13,7 @@ from cognite.neat._rules._shared import (
     Rules,
     VerifiedRules,
 )
+from cognite.neat._store._provenance import Agent as ProvenanceAgent
 
 T_RulesIn = TypeVar("T_RulesIn", bound=Rules)
 T_RulesOut = TypeVar("T_RulesOut", bound=Rules)
@@ -54,6 +56,11 @@ class RulesTransformer(ABC, Generic[T_RulesIn, T_RulesOut]):
         else:
             raise NeatTypeError(f"Unsupported type: {type(rules)}")
 
+    @property
+    def agent(self) -> ProvenanceAgent:
+        """Provenance agent for the importer."""
+        return ProvenanceAgent(id_=DEFAULT_NAMESPACE[f"agent/{type(self).__name__}"])
+
 
 class RulesPipeline(list, MutableSequence[RulesTransformer], Generic[T_RulesIn, T_RulesOut]):
     def transform(self, rules: T_RulesIn | OutRules[T_RulesIn]) -> OutRules[T_RulesOut]:

cognite/neat/_rules/transformers/_converters.py

@@ -10,7 +10,11 @@ from cognite.client.data_classes import data_modeling as dms
 from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
 from rdflib import Namespace
 
-from cognite.neat._constants import COGNITE_MODELS, DMS_CONTAINER_PROPERTY_SIZE_LIMIT
+from cognite.neat._constants import (
+    COGNITE_MODELS,
+    DMS_CONTAINER_PROPERTY_SIZE_LIMIT,
+)
+from cognite.neat._issues._base import IssueList
 from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._issues.warnings._models import (
     EnterpriseModelNotBuildOnTopOfCDMWarning,
@@ -18,7 +22,13 @@ from cognite.neat._issues.warnings._models import (
 )
 from cognite.neat._issues.warnings.user_modeling import ParentInDifferentSpaceWarning
 from cognite.neat._rules._constants import EntityTypes
-from cognite.neat._rules._shared import InputRules, JustRules, OutRules, VerifiedRules
+from cognite.neat._rules._shared import (
+    InputRules,
+    JustRules,
+    OutRules,
+    ReadRules,
+    VerifiedRules,
+)
 from cognite.neat._rules.analysis import DMSAnalysis
 from cognite.neat._rules.models import (
     AssetRules,
@@ -83,13 +93,16 @@ class ToCompliantEntities(RulesTransformer[InformationInputRules, InformationInp
 
     def transform(
         self, rules: InformationInputRules | OutRules[InformationInputRules]
-    ) -> OutRules[InformationInputRules]:
-        return JustRules(self._transform(self._to_rules(rules)))
+    ) -> ReadRules[InformationInputRules]:
+        return ReadRules(self._transform(self._to_rules(rules)), IssueList(), {})
 
     def _transform(self, rules: InformationInputRules) -> InformationInputRules:
         rules.metadata.prefix = self._fix_entity(rules.metadata.prefix)
         rules.classes = self._fix_classes(rules.classes)
         rules.properties = self._fix_properties(rules.properties)
+
+        rules.metadata.version += "_dms_compliant"
+
         return rules
 
     @classmethod
@@ -258,6 +271,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
         type_: Literal["enterprise", "solution"] = "enterprise",
         mode: Literal["read", "write"] = "read",
         dummy_property: str = "GUID",
+        move_connections: bool = False,
     ):
         self.new_model_id = DataModelId.load(new_model_id)
         if not self.new_model_id.version:
@@ -267,6 +281,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
         self.mode = mode
         self.type_ = type_
         self.dummy_property = dummy_property
+        self.move_connections = move_connections
 
     def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
         # Copy to ensure immutability
@@ -393,10 +408,18 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
         # extending reference views with new ones
         enterprise_model.views.extend(enterprise_views)
 
+        # Move connections from reference model to enterprise model
+        if self.move_connections:
+            enterprise_connections = self._move_connections(enterprise_model)
+        else:
+            enterprise_connections = SheetList[DMSProperty]()
+
         # while overwriting containers and properties with new ones
         enterprise_model.containers = enterprise_containers
         enterprise_model.properties = enterprise_properties
 
+        enterprise_properties.extend(enterprise_connections)
+
         return JustRules(enterprise_model)
 
     def _remove_cognite_affix(self, entity: _T_Entity) -> _T_Entity:
@@ -455,6 +478,35 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
 
         return new_views, new_containers, new_properties
 
+    def _move_connections(self, rules: DMSRules) -> SheetList[DMSProperty]:
+        implements: dict[ViewEntity, list[ViewEntity]] = defaultdict(list)
+        new_properties = SheetList[DMSProperty]()
+
+        for view in rules.views:
+            if view.view.space == rules.metadata.space and view.implements:
+                for implemented_view in view.implements:
+                    implements.setdefault(implemented_view, []).append(view.view)
+
+        # currently only supporting single implementation of reference view in enterprise view
+        # connections that do not have properties
+        if all(len(v) == 1 for v in implements.values()):
+            for prop_ in rules.properties:
+                if (
+                    prop_.view.space != rules.metadata.space
+                    and prop_.connection
+                    and isinstance(prop_.value_type, ViewEntity)
+                    and implements.get(prop_.view)
+                    and implements.get(prop_.value_type)
+                ):
+                    if isinstance(prop_.connection, EdgeEntity) and prop_.connection.properties:
+                        continue
+                    new_property = prop_.model_copy(deep=True)
+                    new_property.view = implements[prop_.view][0]
+                    new_property.value_type = implements[prop_.value_type][0]
+                    new_properties.append(new_property)
+
+        return new_properties
+
 
 class ReduceCogniteModel(RulesTransformer[DMSRules, DMSRules]):
     _ASSET_VIEW = ViewId("cdf_cdm", "CogniteAsset", "v1")

cognite/neat/_session/_base.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timezone
 from typing import Literal, cast
 
 from cognite.client import CogniteClient
@@ -14,6 +15,10 @@ from cognite.neat._rules.models.entities._single_value import UnknownEntity
 from cognite.neat._rules.models.information._rules import InformationRules
 from cognite.neat._rules.models.information._rules_input import InformationInputRules
 from cognite.neat._rules.transformers import ConvertToRules, VerifyAnyRules
+from cognite.neat._store._provenance import (
+    INSTANCES_ENTITY,
+    Change,
+)
 
 from ._inspect import InspectAPI
 from ._prepare import PrepareAPI
@@ -22,7 +27,8 @@ from ._set import SetAPI
 from ._show import ShowAPI
 from ._state import SessionState
 from ._to import ToAPI
-from .exceptions import intercept_session_exceptions
+from .engine import load_neat_engine
+from .exceptions import NeatSessionError, intercept_session_exceptions
 
 
 @intercept_session_exceptions
@@ -32,6 +38,7 @@ class NeatSession:
         client: CogniteClient | None = None,
         storage: Literal["memory", "oxigraph"] = "memory",
         verbose: bool = True,
+        load_engine: Literal["newest", "cache", "skip"] = "cache",
     ) -> None:
         self._client = client
         self._verbose = verbose
@@ -42,26 +49,69 @@ class NeatSession:
         self.show = ShowAPI(self._state)
         self.set = SetAPI(self._state, verbose)
         self.inspect = InspectAPI(self._state)
+        if load_engine != "skip" and (engine_version := load_neat_engine(client, load_engine)):
+            print(f"Neat Engine {engine_version} loaded.")
 
     @property
     def version(self) -> str:
         return _version.__version__
 
     def verify(self) -> IssueList:
-        output = VerifyAnyRules("continue").try_transform(self._state.input_rule)
+        source_id, last_unverified_rule = self._state.data_model.last_unverified_rule
+        transformer = VerifyAnyRules("continue")
+        start = datetime.now(timezone.utc)
+        output = transformer.try_transform(last_unverified_rule)
+        end = datetime.now(timezone.utc)
+
         if output.rules:
-            self._state.verified_rules.append(output.rules)
+            change = Change.from_rules_activity(
+                output.rules,
+                transformer.agent,
+                start,
+                end,
+                f"Verified data model {source_id} as {output.rules.id_}",
+                self._state.data_model.provenance.source_entity(source_id)
+                or self._state.data_model.provenance.target_entity(source_id),
+            )
+
+            self._state.data_model.write(output.rules, change)
+
             if isinstance(output.rules, InformationRules):
-                self._state.store.add_rules(output.rules)
+                self._state.instances.store.add_rules(output.rules)
+
         output.issues.action = "verify"
-        self._state.issue_lists.append(output.issues)
+        self._state.data_model.issue_lists.append(output.issues)
         if output.issues:
             print("You can inspect the issues with the .inspect.issues(...) method.")
         return output.issues
 
-    def convert(self, target: Literal["dms"]) -> None:
-        converted = ConvertToRules(DMSRules).transform(self._state.last_verified_rule)
-        self._state.verified_rules.append(converted.rules)
+    def convert(self, target: Literal["dms", "information"]) -> None:
+        start = datetime.now(timezone.utc)
+        if target == "dms":
+            source_id, info_rules = self._state.data_model.last_verified_information_rules
+            converter = ConvertToRules(DMSRules)
+            converted_rules = converter.transform(info_rules).rules
+        elif target == "information":
+            source_id, dms_rules = self._state.data_model.last_verified_dms_rules
+            converter = ConvertToRules(InformationRules)
+            converted_rules = converter.transform(dms_rules).rules
+        else:
+            raise NeatSessionError(f"Target {target} not supported.")
+        end = datetime.now(timezone.utc)
+
+        # Provenance
+        change = Change.from_rules_activity(
+            converted_rules,
+            converter.agent,
+            start,
+            end,
+            f"Converted data model {source_id} to {converted_rules.id_}",
+            self._state.data_model.provenance.source_entity(source_id)
+            or self._state.data_model.provenance.target_entity(source_id),
+        )
+
+        self._state.data_model.write(converted_rules, change)
+
         if self._verbose:
             print(f"Rules converted to {target}")
 
@@ -73,6 +123,7 @@ class NeatSession:
             "v1",
         ),
         non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
+        max_number_of_instance: int = 100,
     ) -> IssueList:
         """Data model inference from instances.
 
@@ -83,35 +134,55 @@ class NeatSession:
 
         model_id = dm.DataModelId.load(model_id)
 
-        input_rules: ReadRules = importers.InferenceImporter.from_graph_store(
-            store=self._state.store,
+        start = datetime.now(timezone.utc)
+        importer = importers.InferenceImporter.from_graph_store(
+            store=self._state.instances.store,
             non_existing_node_type=non_existing_node_type,
-        ).to_rules()
+            max_number_of_instance=max_number_of_instance,
+        )
+        inferred_rules: ReadRules = importer.to_rules()
+        end = datetime.now(timezone.utc)
 
         if model_id.space:
-            cast(InformationInputRules, input_rules.rules).metadata.prefix = model_id.space
+            cast(InformationInputRules, inferred_rules.rules).metadata.prefix = model_id.space
         if model_id.external_id:
-            cast(InformationInputRules, input_rules.rules).metadata.name = model_id.external_id
+            cast(InformationInputRules, inferred_rules.rules).metadata.name = model_id.external_id
 
         if model_id.version:
-            cast(InformationInputRules, input_rules.rules).metadata.version = model_id.version
+            cast(InformationInputRules, inferred_rules.rules).metadata.version = model_id.version
+
+        # Provenance
+        change = Change.from_rules_activity(
+            inferred_rules,
+            importer.agent,
+            start,
+            end,
+            "Inferred data model",
+            INSTANCES_ENTITY,
+        )
 
-        self.read.rdf._store_rules(self._state.store, input_rules, "Data Model Inference")
-        return input_rules.issues
+        self._state.data_model.write(inferred_rules, change)
+        return inferred_rules.issues
 
     def _repr_html_(self) -> str:
         state = self._state
-        if not state.has_store and not state.input_rules:
+        if (
+            not state.instances.has_store
+            and not state.data_model.has_unverified_rules
+            and not state.data_model.has_verified_rules
+        ):
             return "<strong>Empty session</strong>. Get started by reading something with the <em>.read</em> attribute."
 
         output = []
-        if state.input_rules and not state.verified_rules:
-            output.append(f"<H2>Unverified Data Model</H2><br />{state.input_rule.rules._repr_html_()}")  # type: ignore
 
-        if state.verified_rules:
-            output.append(f"<H2>Verified Data Model</H2><br />{state.last_verified_rule._repr_html_()}")  # type: ignore
+        if state.data_model.has_unverified_rules and not state.data_model.has_verified_rules:
+            rules: ReadRules = state.data_model.last_unverified_rule[1]
+            output.append(f"<H2>Unverified Data Model</H2><br />{rules.rules._repr_html_()}")  # type: ignore
+
+        if state.data_model.has_verified_rules:
+            output.append(f"<H2>Verified Data Model</H2><br />{state.data_model.last_verified_rule[1]._repr_html_()}")  # type: ignore
 
-        if state.has_store:
-            output.append(f"<H2>Instances</H2> {state.store._repr_html_()}")
+        if state.instances.has_store:
+            output.append(f"<H2>Instances</H2> {state.instances.store._repr_html_()}")
 
         return "<br />".join(output)

cognite/neat/_session/_inspect.py

@@ -1,4 +1,5 @@
 import difflib
+from collections.abc import Callable
 from typing import Literal, overload
 
 import pandas as pd
@@ -21,7 +22,7 @@ class InspectAPI:
     @property
     def properties(self) -> pd.DataFrame:
         """Returns the properties of the current data model."""
-        return self._state.last_verified_rule.properties.to_pandas()
+        return self._state.data_model.last_verified_rule[1].properties.to_pandas()
 
 
 @intercept_session_exceptions
@@ -51,7 +52,7 @@ class InspectIssues:
         return_dataframe: bool = (False if IN_NOTEBOOK else True),  # type: ignore[assignment]
     ) -> pd.DataFrame | None:
         """Returns the issues of the current data model."""
-        issues = self._state.last_issues
+        issues = self._state.data_model.last_issues
         if not issues:
             self._print("No issues found.")
 
@@ -94,7 +95,14 @@ class InspectIssues:
 @intercept_session_exceptions
 class InspectOutcome:
     def __init__(self, state: SessionState) -> None:
-        self._state = state
+        self.data_model = InspectUploadOutcome(lambda: state.data_model.last_outcome)
+        self.instances = InspectUploadOutcome(lambda: state.instances.last_outcome)
+
+
+@intercept_session_exceptions
+class InspectUploadOutcome:
+    def __init__(self, get_last_outcome: Callable[[], UploadResultList]) -> None:
+        self._get_last_outcome = get_last_outcome
 
     @staticmethod
     def _as_set(value: str | list[str] | None) -> set[str] | None:
@@ -130,7 +138,7 @@ class InspectOutcome:
         return_dataframe: bool = (False if IN_NOTEBOOK else True),  # type: ignore[assignment]
     ) -> pd.DataFrame | None:
         """Returns the outcome of the last upload."""
-        outcome = self._state.last_outcome
+        outcome = self._get_last_outcome()
         name_set = self._as_set(name)
 
         def outcome_filter(item: UploadResultCore) -> bool: