cognite-neat 0.96.6__py3-none-any.whl → 0.97.1__py3-none-any.whl

cognite/neat/_session/_show.py

@@ -5,6 +5,7 @@ from typing import Any, cast
 import networkx as nx
 from IPython.display import HTML, display
 from pyvis.network import Network as PyVisNetwork  # type: ignore
+from rdflib import URIRef
 
 from cognite.neat._constants import IN_NOTEBOOK, IN_PYODIDE
 from cognite.neat._rules._constants import EntityTypes
@@ -67,19 +68,28 @@ class ShowDataModelAPI(ShowBaseAPI):
     def __init__(self, state: SessionState) -> None:
         super().__init__(state)
         self._state = state
+        self.provenance = ShowDataModelProvenanceAPI(self._state)
+        self.implements = ShowDataModelImplementsAPI(self._state)
 
     def __call__(self) -> Any:
-        if not self._state.has_verified_rules:
+        if not self._state.data_model.has_verified_rules:
             raise NeatSessionError(
                 "No verified data model available. Try using [bold].verify()[/bold] to verify data model."
             )
 
-        try:
-            di_graph = self._generate_dms_di_graph(self._state.last_verified_dms_rules)
+        rules = self._state.data_model.last_verified_rule[1]
+
+        if isinstance(rules, DMSRules):
+            di_graph = self._generate_dms_di_graph(self._state.data_model.last_verified_dms_rules[1])
             name = "dms_data_model.html"
-        except NeatSessionError:
-            di_graph = self._generate_info_di_graph(self._state.last_verified_information_rules)
+        elif isinstance(rules, InformationRules):
+            di_graph = self._generate_info_di_graph(self._state.data_model.last_verified_information_rules[1])
             name = "information_data_model.html"
+        else:
+            # This should never happen, but we need to handle it to satisfy mypy
+            raise NeatSessionError(
+                f"Unsupported type {type(rules) }. Make sure you have either information or DMS rules."
+            )
 
         return self._generate_visualization(di_graph, name)
 
@@ -91,20 +101,7 @@ class ShowDataModelAPI(ShowBaseAPI):
         for view in rules.views:
             # if possible use human readable label coming from the view name
             if not di_graph.has_node(view.view.suffix):
-                di_graph.add_node(view.view.suffix, label=view.name or view.view.suffix)
-
-            # add implements as edges
-            if view.implements:
-                for implement in view.implements:
-                    if not di_graph.has_node(implement.suffix):
-                        di_graph.add_node(implement.suffix, label=implement.suffix)
-
-                    di_graph.add_edge(
-                        view.view.suffix,
-                        implement.suffix,
-                        label="implements",
-                        dashes=True,
-                    )
+                di_graph.add_node(view.view.suffix, label=view.view.suffix)
 
         # Add nodes and edges from Properties sheet
         for prop_ in rules.properties:
@@ -133,8 +130,89 @@ class ShowDataModelAPI(ShowBaseAPI):
                     label=class_.name or class_.class_.suffix,
                 )
 
+        # Add nodes and edges from Properties sheet
+        for prop_ in rules.properties:
+            if prop_.type_ == EntityTypes.object_property:
+                if not di_graph.has_node(prop_.class_.suffix):
+                    di_graph.add_node(prop_.class_.suffix, label=prop_.class_.suffix)
+
+                di_graph.add_edge(
+                    prop_.class_.suffix,
+                    cast(ClassEntity, prop_.value_type).suffix,
+                    label=prop_.name or prop_.property_,
+                )
+
+        return di_graph
+
+
+@intercept_session_exceptions
+class ShowDataModelImplementsAPI(ShowBaseAPI):
+    def __init__(self, state: SessionState) -> None:
+        super().__init__(state)
+        self._state = state
+
+    def __call__(self) -> Any:
+        if not self._state.data_model.has_verified_rules:
+            raise NeatSessionError(
+                "No verified data model available. Try using [bold].verify()[/bold] to verify data model."
+            )
+
+        rules = self._state.data_model.last_verified_rule[1]
+
+        if isinstance(rules, DMSRules):
+            di_graph = self._generate_dms_di_graph(self._state.data_model.last_verified_dms_rules[1])
+            name = "dms_data_model_implements.html"
+        elif isinstance(rules, InformationRules):
+            di_graph = self._generate_info_di_graph(self._state.data_model.last_verified_information_rules[1])
+            name = "information_data_model_implements.html"
+        else:
+            # This should never happen, but we need to handle it to satisfy mypy
+            raise NeatSessionError(
+                f"Unsupported type {type(rules) }. Make sure you have either information or DMS rules."
+            )
+
+        return self._generate_visualization(di_graph, name)
+
+    def _generate_dms_di_graph(self, rules: DMSRules) -> nx.DiGraph:
+        """Generate a DiGraph from the last verified DMS rules."""
+        di_graph = nx.DiGraph()
+
+        # Add nodes and edges from Views sheet
+        for view in rules.views:
+            # add implements as edges
+            if view.implements:
+                if not di_graph.has_node(view.view.suffix):
+                    di_graph.add_node(view.view.suffix, label=view.view.suffix)
+                for implement in view.implements:
+                    if not di_graph.has_node(implement.suffix):
+                        di_graph.add_node(implement.suffix, label=implement.suffix)
+
+                    di_graph.add_edge(
+                        view.view.suffix,
+                        implement.suffix,
+                        label="implements",
+                        dashes=True,
+                    )
+
+        return di_graph
+
+    def _generate_info_di_graph(self, rules: InformationRules) -> nx.DiGraph:
+        """Generate DiGraph representing information data model."""
+
+        di_graph = nx.DiGraph()
+
+        # Add nodes and edges from Views sheet
+        for class_ in rules.classes:
+            # if possible use human readable label coming from the view name
+
             # add subClassOff as edges
             if class_.parent:
+                if not di_graph.has_node(class_.class_.suffix):
+                    di_graph.add_node(
+                        class_.class_.suffix,
+                        label=class_.name or class_.class_.suffix,
+                    )
+
                 for parent in class_.parent:
                     if not di_graph.has_node(parent.suffix):
                         di_graph.add_node(parent.suffix, label=parent.suffix)
@@ -145,20 +223,68 @@ class ShowDataModelAPI(ShowBaseAPI):
                         dashes=True,
                     )
 
-        # Add nodes and edges from Properties sheet
-        for prop_ in rules.properties:
-            if prop_.type_ == EntityTypes.object_property:
-                if not di_graph.has_node(prop_.class_.suffix):
-                    di_graph.add_node(prop_.class_.suffix, label=prop_.class_.suffix)
+        return di_graph
 
-                di_graph.add_edge(
-                    prop_.class_.suffix,
-                    cast(ClassEntity, prop_.value_type).suffix,
-                    label=prop_.name or prop_.property_,
-                )
+
+@intercept_session_exceptions
+class ShowDataModelProvenanceAPI(ShowBaseAPI):
+    def __init__(self, state: SessionState) -> None:
+        super().__init__(state)
+        self._state = state
+
+    def __call__(self) -> Any:
+        if not self._state.data_model.provenance:
+            raise NeatSessionError("No data model available. Try using [bold].read[/bold] to load data model.")
+
+        di_graph = self._generate_dm_provenance_di_graph_and_types()
+        return self._generate_visualization(di_graph, name="data_model_provenance.html")
+
+    def _generate_dm_provenance_di_graph_and_types(self) -> nx.DiGraph:
+        di_graph = nx.DiGraph()
+        hex_colored_types = _generate_hex_color_per_type(["Agent", "Entity", "Activity"])
+
+        for change in self._state.data_model.provenance:
+            source = self._shorten_id(change.source_entity.id_)
+            target = self._shorten_id(change.target_entity.id_)
+            agent = self._shorten_id(change.agent.id_)
+
+            di_graph.add_node(
+                source,
+                label=source,
+                type="Entity",
+                title="Entity",
+                color=hex_colored_types["Entity"],
+            )
+
+            di_graph.add_node(
+                target,
+                label=target,
+                type="Entity",
+                title="Entity",
+                color=hex_colored_types["Entity"],
+            )
+
+            di_graph.add_node(
+                agent,
+                label=agent,
+                type="Agent",
+                title="Agent",
+                color=hex_colored_types["Agent"],
+            )
+
+            di_graph.add_edge(source, agent, label="used", color="grey")
+            di_graph.add_edge(agent, target, label="generated", color="grey")
 
         return di_graph
 
+    @staticmethod
+    def _shorten_id(thing: URIRef) -> str:
+        if "https://cognitedata.com/dms/data-model/" in thing:
+            return "DMS(" + ",".join(thing.replace("https://cognitedata.com/dms/data-model/", "").split("/")) + ")"
+        elif "http://purl.org/cognite/neat/data-model/" in thing:
+            return "NEAT(" + ",".join(thing.replace("http://purl.org/cognite/neat/data-model/", "").split("/")) + ")"
+        return remove_namespace_from_uri(thing)
+
 
 @intercept_session_exceptions
 class ShowInstanceAPI(ShowBaseAPI):
@@ -167,7 +293,20 @@ class ShowInstanceAPI(ShowBaseAPI):
         self._state = state
 
     def __call__(self) -> Any:
-        if not self._state.store.graph:
+        if IN_PYODIDE:
+            raise NeatSessionError(
+                "Instances visualization not available if neat is run in PYODIDE as "
+                "PYODIDE dues not support oxigraph storage for NeatSession."
+                "Try running neat in regular Jupyter notebook and set [bold]NeatSession(storage='oxigraph')[/bold]."
+            )
+
+        if not self._state.instances.store_type == "oxigraph":
+            raise NeatSessionError(
+                "Visualization is only available for Oxigraph store. "
+                'Try setting [bold]NeatSession(storage="oxigraph")[/bold] enable Oxigraph store.'
+            )
+
+        if not self._state.instances.store.graph:
             raise NeatSessionError("No instances available. Try using [bold].read[/bold] to load instances.")
 
         di_graph = self._generate_instance_di_graph_and_types()
@@ -189,8 +328,8 @@ class ShowInstanceAPI(ShowBaseAPI):
 
         di_graph = nx.DiGraph()
 
-        types = [type_ for type_, _ in self._state.store.queries.summarize_instances()]
-        hex_colored_types = self._generate_hex_color_per_type(types)
+        types = [type_ for type_, _ in self._state.instances.store.queries.summarize_instances()]
+        hex_colored_types = _generate_hex_color_per_type(types)
 
         for (  # type: ignore
             subject,
@@ -198,7 +337,7 @@ class ShowInstanceAPI(ShowBaseAPI):
             object,
             subject_type,
             object_type,
-        ) in self._state.store.graph.query(query):
+        ) in self._state.instances.store.graph.query(query):
             subject = remove_namespace_from_uri(subject)
             property_ = remove_namespace_from_uri(property_)
             object = remove_namespace_from_uri(object)
@@ -223,15 +362,15 @@ class ShowInstanceAPI(ShowBaseAPI):
 
         return di_graph
 
-    @staticmethod
-    def _generate_hex_color_per_type(types: list[str]) -> dict:
-        hex_colored_types = {}
-        random.seed(381)
-        for type_ in types:
-            hue = random.random()
-            saturation = random.uniform(0.5, 1.0)
-            lightness = random.uniform(0.4, 0.6)
-            rgb = colorsys.hls_to_rgb(hue, lightness, saturation)
-            hex_color = f"#{int(rgb[0] * 255):02x}{int(rgb[1] * 255):02x}{int(rgb[2] * 255):02x}"
-            hex_colored_types[type_] = hex_color
-        return hex_colored_types
+
+def _generate_hex_color_per_type(types: list[str]) -> dict:
+    hex_colored_types = {}
+    random.seed(381)
+    for type_ in types:
+        hue = random.random()
+        saturation = random.uniform(0.5, 1.0)
+        lightness = random.uniform(0.4, 0.6)
+        rgb = colorsys.hls_to_rgb(hue, lightness, saturation)
+        hex_color = f"#{int(rgb[0] * 255):02x}{int(rgb[1] * 255):02x}{int(rgb[2] * 255):02x}"
+        hex_colored_types[type_] = hex_color
+    return hex_colored_types

cognite/neat/_session/_state.py

@@ -1,22 +1,29 @@
 from dataclasses import dataclass, field
 from typing import Literal, cast
 
+from rdflib import URIRef
+
 from cognite.neat._issues import IssueList
-from cognite.neat._rules._shared import ReadRules, VerifiedRules
+from cognite.neat._rules._shared import JustRules, ReadRules, VerifiedRules
 from cognite.neat._rules.models.dms._rules import DMSRules
 from cognite.neat._rules.models.information._rules import InformationRules
 from cognite.neat._rules.models.information._rules_input import InformationInputRules
 from cognite.neat._store import NeatGraphStore
+from cognite.neat._store._provenance import Change, Provenance
 from cognite.neat._utils.upload import UploadResultList
 
 from .exceptions import NeatSessionError
 
 
-@dataclass
 class SessionState:
+    def __init__(self, store_type: Literal["memory", "oxigraph"]) -> None:
+        self.instances = InstancesState(store_type)
+        self.data_model = DataModelState()
+
+
+@dataclass
+class InstancesState:
     store_type: Literal["memory", "oxigraph"]
-    input_rules: list[ReadRules] = field(default_factory=list)
-    verified_rules: list[VerifiedRules] = field(default_factory=list)
     issue_lists: list[IssueList] = field(default_factory=list)
     outcome: list[UploadResultList] = field(default_factory=list)
     _store: NeatGraphStore | None = field(init=False, default=None)
@@ -31,62 +38,96 @@ class SessionState:
         return cast(NeatGraphStore, self._store)
 
     @property
-    def input_rule(self) -> ReadRules:
-        if not self.input_rules:
-            raise NeatSessionError("No input data model available. Try using [bold].read[/bold] to load a data model.")
-        return self.input_rules[-1]
+    def has_store(self) -> bool:
+        return self._store is not None
 
     @property
     def last_outcome(self) -> UploadResultList:
         if not self.outcome:
             raise NeatSessionError(
-                "No outcome available. Try using [bold].to.cdf[/bold] to upload a data model/instances."
+                "No outcome available. Try using [bold].to.cdf.instances[/bold] to upload a data minstances."
             )
         return self.outcome[-1]
 
+
+@dataclass
+class DataModelState:
+    _rules: dict[URIRef, ReadRules | JustRules | VerifiedRules] = field(init=False, default_factory=dict)
+    issue_lists: list[IssueList] = field(default_factory=list)
+    provenance: Provenance = field(default_factory=Provenance)
+    outcome: list[UploadResultList] = field(default_factory=list)
+
+    def write(self, rules: ReadRules | JustRules | VerifiedRules, change: Change) -> None:
+        if change.target_entity.id_ in self._rules:
+            raise NeatSessionError(f"Data model <{change.target_entity.id_}> already exists.")
+
+        else:
+            self._rules[change.target_entity.id_] = rules
+            self.provenance.append(change)
+
+    def read(self, id_: URIRef) -> ReadRules | JustRules | VerifiedRules:
+        if id_ not in self._rules:
+            raise NeatSessionError(f"Data model <{id_}> not found.")
+        return self._rules[id_]
+
+    @property
+    def unverified_rules(self) -> dict[URIRef, ReadRules]:
+        return {id_: rules for id_, rules in self._rules.items() if isinstance(rules, ReadRules)}
+
     @property
-    def information_input_rule(self) -> ReadRules | None:
-        if self.input_rules:
-            for rule in self.input_rules[::-1]:
+    def verified_rules(self) -> dict[URIRef, VerifiedRules]:
+        return {id_: rules for id_, rules in self._rules.items() if isinstance(rules, VerifiedRules)}
+
+    @property
+    def last_unverified_rule(self) -> tuple[URIRef, ReadRules]:
+        if not self.unverified_rules:
+            raise NeatSessionError("No data model available. Try using [bold].read[/bold] to load a data model.")
+        return next(reversed(self.unverified_rules.items()))
+
+    @property
+    def last_info_unverified_rule(self) -> tuple[URIRef, ReadRules]:
+        if self.unverified_rules:
+            for id_, rule in reversed(self.unverified_rules.items()):
                 if isinstance(rule.rules, InformationInputRules):
-                    return rule
-        return None
+                    return id_, rule
+
+        raise NeatSessionError("No data model available. Try using [bold].read[/bold] to load a data model.")
 
     @property
-    def last_verified_rule(self) -> VerifiedRules:
+    def last_verified_rule(self) -> tuple[URIRef, VerifiedRules]:
         if not self.verified_rules:
             raise NeatSessionError(
                 "No data model available to verify. Try using  [bold].read[/bold] to load a data model."
             )
-        return self.verified_rules[-1]
+        return next(reversed(self.verified_rules.items()))
 
     @property
-    def last_verified_dms_rules(self) -> DMSRules:
+    def last_verified_information_rules(self) -> tuple[URIRef, InformationRules]:
         if self.verified_rules:
-            for rules in self.verified_rules[::-1]:
-                if isinstance(rules, DMSRules):
-                    return rules
+            for id_, rules in reversed(self.verified_rules.items()):
+                if isinstance(rules, InformationRules):
+                    return id_, rules
 
         raise NeatSessionError(
-            'No verified DMS data model. Try using  [bold].convert("DMS")[/bold]'
-            " to convert verified information model to verified DMS model."
+            "No verified information data model. Try using  [bold].verify()[/bold]"
+            " to convert unverified information model to verified information model."
         )
 
     @property
-    def last_verified_information_rules(self) -> InformationRules:
+    def last_verified_dms_rules(self) -> tuple[URIRef, DMSRules]:
         if self.verified_rules:
-            for rules in self.verified_rules[::-1]:
-                if isinstance(rules, InformationRules):
-                    return rules
+            for id_, rules in reversed(self.verified_rules.items()):
+                if isinstance(rules, DMSRules):
+                    return id_, rules
 
         raise NeatSessionError(
-            "No verified information data model. Try using  [bold].verify()[/bold]"
-            " to convert unverified information model to verified information model."
+            'No verified DMS data model. Try using  [bold].convert("DMS")[/bold]'
+            " to convert verified information model to verified DMS model."
         )
 
     @property
-    def has_store(self) -> bool:
-        return self._store is not None
+    def has_unverified_rules(self) -> bool:
+        return bool(self.unverified_rules)
 
     @property
     def has_verified_rules(self) -> bool:
@@ -97,3 +138,11 @@ class SessionState:
         if not self.issue_lists:
             raise NeatSessionError("No issues available. Try using [bold].verify()[/bold] to verify a data model.")
         return self.issue_lists[-1]
+
+    @property
+    def last_outcome(self) -> UploadResultList:
+        if not self.outcome:
+            raise NeatSessionError(
+                "No outcome available. Try using [bold].to.cdf.data_model[/bold] to upload a data model."
+            )
+        return self.outcome[-1]

cognite/neat/_session/_to.py

@@ -10,7 +10,7 @@ from cognite.neat._session._wizard import space_wizard
 from cognite.neat._utils.upload import UploadResultCore
 
 from ._state import SessionState
-from .exceptions import intercept_session_exceptions
+from .exceptions import NeatSessionError, intercept_session_exceptions
 
 
 @intercept_session_exceptions
@@ -25,7 +25,7 @@ class ToAPI:
         io: Any,
     ) -> None:
         exporter = exporters.ExcelExporter()
-        exporter.export_to_file(self._state.last_verified_rule, Path(io))
+        exporter.export_to_file(self._state.data_model.last_verified_rule[1], Path(io))
         return None
 
     @overload
@@ -37,9 +37,9 @@ class ToAPI:
     def yaml(self, io: Any | None = None) -> str | None:
         exporter = exporters.YAMLExporter()
         if io is None:
-            return exporter.export(self._state.last_verified_rule)
+            return exporter.export(self._state.data_model.last_verified_rule[1])
 
-        exporter.export_to_file(self._state.last_verified_rule, Path(io))
+        exporter.export_to_file(self._state.data_model.last_verified_rule[1], Path(io))
         return None
 
 
@@ -51,16 +51,17 @@ class CDFToAPI:
         self._verbose = verbose
 
     def instances(self, space: str | None = None):
-        if not self._state.last_verified_dms_rules:
-            raise ValueError("No verified DMS data model available")
+        if not self._client:
+            raise NeatSessionError("No CDF client provided!")
 
         loader = loaders.DMSLoader.from_rules(
-            self._state.last_verified_dms_rules, self._state.store, space_wizard(space=space)
+            self._state.data_model.last_verified_dms_rules[1],
+            self._state.instances.store,
+            space_wizard(space=space),
         )
-
-        if not self._client:
-            raise ValueError("No client provided!")
-
+        result = loader.load_into_cdf(self._client)
+        self._state.instances.outcome.append(result)
+        print("You can inspect the details with the .inspect.instances.outcome(...) method.")
         return loader.load_into_cdf(self._client)
 
     def data_model(
@@ -84,20 +85,18 @@ class CDFToAPI:
         - "force": If any component already exists, it will be deleted and recreated.
 
         """
-        if not self._state.last_verified_dms_rules:
-            raise ValueError("No verified DMS data model available")
 
         exporter = exporters.DMSExporter(existing_handling=existing_handling)
 
         if not self._client:
-            raise ValueError("No client provided!")
+            raise NeatSessionError("No client provided!")
 
         conversion_issues = IssueList(action="to.cdf.data_model")
         with catch_warnings(conversion_issues):
             result = exporter.export_to_cdf(
-                self._state.last_verified_dms_rules, self._client, dry_run, fallback_one_by_one
+                self._state.data_model.last_verified_dms_rules[1], self._client, dry_run, fallback_one_by_one
             )
         result.insert(0, UploadResultCore(name="schema", issues=conversion_issues))
-        self._state.outcome.append(result)
-        print("You can inspect the details with the .inspect.outcome(...) method.")
+        self._state.data_model.outcome.append(result)
+        print("You can inspect the details with the .inspect.data_model.outcome(...) method.")
         return result

cognite/neat/_session/engine/__init__.py

@@ -0,0 +1,4 @@
+from ._import import import_engine
+from ._load import load_neat_engine
+
+__all__ = ["load_neat_engine", "import_engine"]

cognite/neat/_session/engine/_import.py

@@ -0,0 +1,7 @@
+from ._interface import NeatEngine
+
+
+def import_engine() -> NeatEngine:
+    from neatengine import NeatEngine  # type: ignore[import-not-found]
+
+    return NeatEngine()

cognite/neat/_session/engine/_interface.py

@@ -0,0 +1,24 @@
+from collections.abc import Iterable
+from typing import Any, Protocol
+
+from rdflib import Literal, URIRef
+
+
+class Extractor(Protocol):
+    def extract(self) -> Iterable[tuple[URIRef, URIRef, Literal | URIRef]]: ...
+
+
+class ConfigAPI(Protocol):
+    source: str | None
+    file: Any | None
+    type: str | None
+    primary_key: str | None
+
+
+class NeatEngine(Protocol):
+    version: str = "1.0.0"
+
+    @property
+    def set(self) -> ConfigAPI: ...
+
+    def create_extractor(self) -> Extractor: ...