cognite-neat 0.92.2__py3-none-any.whl → 0.93.0__py3-none-any.whl

cognite/neat/{rules → _rules}/transformers/_pipelines.py

@@ -1,9 +1,9 @@
 from collections.abc import Iterable
 
-from cognite.neat.issues.errors import NeatValueError
-from cognite.neat.rules._shared import InputRules, MaybeRules, VerifiedRules
-from cognite.neat.rules.importers import BaseImporter
-from cognite.neat.rules.models import VERIFIED_RULES_BY_ROLE, RoleTypes
+from cognite.neat._issues.errors import NeatValueError
+from cognite.neat._rules._shared import InputRules, MaybeRules, VerifiedRules
+from cognite.neat._rules.importers import BaseImporter
+from cognite.neat._rules.models import VERIFIED_RULES_BY_ROLE, RoleTypes
 
 from ._base import RulesPipeline, RulesTransformer
 from ._converters import ConvertToRules

cognite/neat/{rules → _rules}/transformers/_verification.py

@@ -6,9 +6,9 @@ from typing import Any, Literal
 
 from pydantic import ValidationError
 
-from cognite.neat.issues import IssueList, NeatError, NeatWarning
-from cognite.neat.issues.errors import NeatTypeError
-from cognite.neat.rules._shared import (
+from cognite.neat._issues import IssueList, MultiValueError, NeatError, NeatWarning
+from cognite.neat._issues.errors import NeatTypeError
+from cognite.neat._rules._shared import (
     InputRules,
     MaybeRules,
     OutRules,
@@ -17,7 +17,7 @@ from cognite.neat.rules._shared import (
     T_VerifiedRules,
     VerifiedRules,
 )
-from cognite.neat.rules.models import (
+from cognite.neat._rules.models import (
     AssetInputRules,
     AssetRules,
     DMSInputRules,
@@ -129,6 +129,12 @@ def _handle_issues(
         except ValidationError as e:
             issues.extend(error_cls.from_pydantic_errors(e.errors(), **(error_args or {})))
             future_result._result = "failure"
+        except MultiValueError as e:
+            issues.extend(e.errors)
+            future_result._result = "failure"
+        except NeatError as e:
+            issues.append(e)
+            future_result._result = "failure"
         else:
             future_result._result = "success"
         finally:

cognite/neat/_session/__init__.py

@@ -0,0 +1,3 @@
+from ._base import NeatSession
+
+__all__ = ["NeatSession"]

cognite/neat/_session/_base.py

@@ -0,0 +1,86 @@
+from typing import Literal, cast
+
+import pandas as pd
+from cognite.client import CogniteClient
+
+from cognite.neat._issues import IssueList
+from cognite.neat._rules import importers
+from cognite.neat._rules._shared import ReadRules
+from cognite.neat._rules.models import DMSRules
+from cognite.neat._rules.models._base_input import InputComponent
+from cognite.neat._rules.models.information._rules import InformationRules
+from cognite.neat._rules.models.information._rules_input import InformationInputRules
+from cognite.neat._rules.transformers import ConvertToRules, VerifyAnyRules
+
+from ._prepare import PrepareAPI
+from ._read import ReadAPI
+from ._show import ShowAPI
+from ._state import SessionState
+from ._to import ToAPI
+from .exceptions import intercept_session_exceptions
+
+
+@intercept_session_exceptions
+class NeatSession:
+    def __init__(
+        self,
+        client: CogniteClient | None = None,
+        storage: Literal["memory", "oxigraph"] = "memory",
+        verbose: bool = True,
+    ) -> None:
+        self._client = client
+        self._verbose = verbose
+        self._state = SessionState(store_type=storage)
+        self.read = ReadAPI(self._state, client, verbose)
+        self.to = ToAPI(self._state, client, verbose)
+        self.prepare = PrepareAPI(self._state, verbose)
+        self.show = ShowAPI(self._state)
+
+    def verify(self) -> IssueList:
+        output = VerifyAnyRules("continue").try_transform(self._state.input_rule)
+        if output.rules:
+            self._state.verified_rules.append(output.rules)
+            if isinstance(output.rules, InformationRules):
+                self._state.store.add_rules(output.rules)
+        return output.issues
+
+    def convert(self, target: Literal["dms"]) -> None:
+        converted = ConvertToRules(DMSRules).transform(self._state.last_verified_rule)
+        self._state.verified_rules.append(converted.rules)
+        if self._verbose:
+            print(f"Rules converted to {target}")
+
+    def infer(
+        self,
+        space: str = "inference_space",
+        external_id: str = "InferredDataModel",
+        version: str = "v1",
+    ) -> IssueList:
+        input_rules: ReadRules = importers.InferenceImporter.from_graph_store(self._state.store).to_rules()
+
+        cast(InformationInputRules, input_rules.rules).metadata.prefix = space
+        cast(InformationInputRules, input_rules.rules).metadata.name = external_id
+        cast(InformationInputRules, input_rules.rules).metadata.version = version
+
+        self.read.rdf._store_rules(self._state.store, input_rules, "Data Model Inference")
+        return input_rules.issues
+
+    def _repr_html_(self) -> str:
+        state = self._state
+        if not state.has_store and not state.input_rules:
+            return "<strong>Empty session</strong>. Get started by reading something with the <em>.read</em> attribute."
+
+        output = []
+        if state.input_rules and not state.verified_rules:
+            metadata = cast(InputComponent, state.input_rule.rules.metadata)  # type: ignore[union-attr]
+            table = pd.DataFrame([metadata.dump()]).T._repr_html_()  # type: ignore[operator]
+            output.append(f"<H2>Raw Data Model</H2><br />{table}")
+
+        if state.verified_rules:
+            table = pd.DataFrame([state.last_verified_rule.metadata.model_dump()]).T._repr_html_()  # type: ignore[operator]
+            output.append(f"<H2>Data Model</H2><br />{table}")
+
+        if state.has_store:
+            output.append(f"<H2>Instances</H2> {state.store._repr_html_()}")
+
+        return "<br />".join(output)

cognite/neat/_session/_prepare.py

@@ -0,0 +1,61 @@
+from collections.abc import Collection
+from typing import Literal, cast
+
+from cognite.client.data_classes.data_modeling import DataModelIdentifier
+
+from cognite.neat._issues._base import IssueList
+from cognite.neat._rules._shared import ReadRules
+from cognite.neat._rules.models.information._rules_input import InformationInputRules
+from cognite.neat._rules.transformers import ReduceCogniteModel, ToCompliantEntities, ToExtension
+
+from ._state import SessionState
+
+
+class PrepareAPI:
+    def __init__(self, state: SessionState, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+        self.data_model = DataModelPrepareAPI(state, verbose)
+
+
+class DataModelPrepareAPI:
+    def __init__(self, state: SessionState, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+
+    def cdf_compliant_external_ids(self) -> None:
+        """Convert data model component external ids to CDF compliant entities."""
+        if input := self._state.information_input_rule:
+            output = ToCompliantEntities().transform(input)
+            self._state.input_rules.append(
+                ReadRules(
+                    rules=cast(InformationInputRules, output.get_rules()),
+                    issues=IssueList(),
+                    read_context={},
+                )
+            )
+
+    def to_extension(self, new_data_model_id: DataModelIdentifier, org_name: str | None = None) -> None:
+        """Uses the current data model as a basis to extend from.
+
+        Args:
+            new_data_model_id: The new data model that is extending the current data model.
+            org_name: Organization name to use for the views in the new data model. This is required if you are
+                creating an extension from a Cognite Data Model.
+
+        """
+        if dms := self._state.last_verified_dms_rules:
+            output = ToExtension(new_data_model_id, org_name).transform(dms)
+            self._state.verified_rules.append(output.rules)
+
+    def reduce(self, drop: Collection[Literal["3D", "Annotation", "BaseViews"]]) -> None:
+        """This is a special method that allow you to drop parts of the data model.
+        This only applies to Cognite Data Models.
+
+        Args:
+            drop: Which parts of the data model to drop.
+
+        """
+        if dms := self._state.last_verified_dms_rules:
+            output = ReduceCogniteModel(drop).transform(dms)
+            self._state.verified_rules.append(output.rules)

cognite/neat/_session/_read.py

@@ -0,0 +1,118 @@
+from pathlib import Path
+from typing import Any
+
+from cognite.client import CogniteClient
+from cognite.client.data_classes.data_modeling import DataModelIdentifier
+
+from cognite.neat._graph import examples as instances_examples
+from cognite.neat._graph import extractors
+from cognite.neat._issues import IssueList
+from cognite.neat._rules import importers
+from cognite.neat._rules._shared import ReadRules
+
+from ._state import SessionState
+from ._wizard import NeatObjectType, RDFFileType, object_wizard, rdf_dm_wizard
+
+
+class ReadAPI:
+    def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+        self.cdf = CDFReadAPI(state, client, verbose)
+        self.rdf = RDFReadAPI(state, client, verbose)
+        self.excel = ExcelReadAPI(state, client, verbose)
+
+
+class BaseReadAPI:
+    def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+        self._client = client
+
+    def _store_rules(self, io: Any, input_rules: ReadRules, source: str) -> None:
+        if input_rules.rules:
+            self._state.input_rules.append(input_rules)
+            if self._verbose:
+                if input_rules.issues.has_errors:
+                    print(f"{source} {type(io)} {io} read failed")
+                else:
+                    print(f"{source} {type(io)} {io} read successfully")
+
+    def _return_filepath(self, io: Any) -> Path:
+        if isinstance(io, str):
+            return Path(io)
+        elif isinstance(io, Path):
+            return io
+        else:
+            raise ValueError(f"Expected str or Path, got {type(io)}")
+
+
+class CDFReadAPI(BaseReadAPI):
+    def data_model(self, data_model_id: DataModelIdentifier) -> IssueList:
+        if self._client is None:
+            raise ValueError("No client provided. Please provide a client to read a data model.")
+
+        importer = importers.DMSImporter.from_data_model_id(self._client, data_model_id)
+        input_rules = importer.to_rules()
+        self._store_rules(data_model_id, input_rules, "CDF")
+        return input_rules.issues
+
+
+class ExcelReadAPI(BaseReadAPI):
+    def __call__(self, io: Any) -> IssueList:
+        filepath = self._return_filepath(io)
+        input_rules: ReadRules = importers.ExcelImporter(filepath).to_rules()
+        self._store_rules(io, input_rules, "Excel")
+        return input_rules.issues
+
+
+class RDFReadAPI(BaseReadAPI):
+    def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
+        super().__init__(state, client, verbose)
+        self.examples = RDFExamples(state)
+
+    def _ontology(self, io: Any) -> IssueList:
+        filepath = self._return_filepath(io)
+        input_rules: ReadRules = importers.OWLImporter.from_file(filepath).to_rules()
+        self._store_rules(io, input_rules, "Ontology")
+        return input_rules.issues
+
+    def _imf(self, io: Any) -> IssueList:
+        filepath = self._return_filepath(io)
+        input_rules: ReadRules = importers.IMFImporter.from_file(filepath).to_rules()
+        self._store_rules(io, input_rules, "IMF Types")
+        return input_rules.issues
+
+    def __call__(
+        self,
+        io: Any,
+        type: NeatObjectType | None = None,
+        source: RDFFileType | None = None,
+    ) -> IssueList:
+        if type is None:
+            type = object_wizard()
+
+        if type.lower() == "Data Model".lower():
+            source = source or rdf_dm_wizard("What type of data model is the RDF?")
+            if source == "Ontology":
+                return self._ontology(io)
+            elif source == "IMF":
+                return self._imf(io)
+            else:
+                raise ValueError(f"Expected ontology, imf or instances, got {source}")
+
+        elif type.lower() == "Instances".lower():
+            self._state.store.write(extractors.RdfFileExtractor(self._return_filepath(io)))
+            return IssueList()
+        else:
+            raise ValueError(f"Expected data model or instances, got {type}")
+
+
+class RDFExamples:
+    def __init__(self, state: SessionState) -> None:
+        self._state = state
+
+    @property
+    def nordic44(self) -> IssueList:
+        self._state.store.write(extractors.RdfFileExtractor(instances_examples.nordic44_knowledge_graph))
+        return IssueList()

cognite/neat/_session/_show.py

@@ -0,0 +1,96 @@
+from typing import Any, cast
+
+import networkx as nx
+from ipycytoscape import CytoscapeWidget  # type: ignore
+from IPython.display import display
+
+from cognite.neat._rules.models.dms._rules import DMSRules
+from cognite.neat._rules.models.entities._single_value import ViewEntity
+
+from ._state import SessionState
+
+
+class ShowAPI:
+    def __init__(self, state: SessionState) -> None:
+        self._state = state
+        self.data_model = ShowDataModelAPI(self._state)
+
+
+class ShowDataModelAPI:
+    def __init__(self, state: SessionState) -> None:
+        self._state = state
+
+    def __call__(self) -> Any:
+        if self._state.last_verified_dms_rules:
+            digraph = self._generate_dms_di_graph()
+            widget = self._generate_widget()
+            widget.graph.add_graph_from_networkx(digraph)
+            return display(widget)
+
+    def _generate_dms_di_graph(self) -> nx.DiGraph:
+        """Generate a DiGraph from the last verified DMS rules."""
+        G = nx.DiGraph()
+
+        nodes, edges = self._generate_dms_rules_nodes_and_edges()
+        G.add_nodes_from(nodes)
+        G.add_edges_from(edges)
+        for node in G.nodes:
+            G.nodes[node]["label"] = node
+
+        return G
+
+    def _generate_dms_rules_nodes_and_edges(self) -> tuple[list[str], list[tuple[str, str, dict]]]:
+        """Generate nodes and edges for the last verified DMS rules for DiGraph."""
+
+        nodes = []
+        edges = []
+
+        for prop_ in cast(DMSRules, self._state.last_verified_dms_rules).properties:
+            nodes.append(prop_.view.suffix)
+
+            if prop_.connection and isinstance(prop_.value_type, ViewEntity):
+                label = f"{prop_.property_} [{0 if prop_.nullable else 1}..{ '' if prop_.is_list else 1}]"
+                edges.append((prop_.view.suffix, prop_.value_type.suffix, {"label": label}))
+
+        for view in cast(DMSRules, self._state.last_verified_dms_rules).views:
+            nodes.append(view.view.suffix)
+
+            if view.implements:
+                for implement in view.implements:
+                    edges.append((view.view.suffix, implement.suffix, {"label": "implements"}))
+
+        return nodes, edges
+
+    def _generate_widget(self):
+        """Generates an empty a CytoscapeWidget."""
+        widget = CytoscapeWidget()
+        widget.layout.height = "500px"
+
+        widget.set_style(
+            [
+                {
+                    "selector": "node",
+                    "css": {
+                        "content": "data(label)",
+                        "text-valign": "center",
+                        "color": "black",
+                        "background-color": "#33C4FF",
+                        "font-size": "10px",
+                        "width": "mapData(score, 0, 1, 10, 50)",
+                        "height": "mapData(score, 0, 1, 10, 50)",
+                    },
+                },
+                {
+                    "selector": "edge",
+                    "style": {
+                        "width": 1,
+                        "target-arrow-shape": "triangle",
+                        "curve-style": "bezier",
+                        "label": "data(label)",
+                        "font-size": "8px",
+                    },
+                },
+            ]
+        )
+
+        return widget

cognite/neat/_session/_state.py

@@ -0,0 +1,69 @@
+from dataclasses import dataclass, field
+from typing import Literal, cast
+
+from cognite.neat._rules._shared import ReadRules, VerifiedRules
+from cognite.neat._rules.models.dms._rules import DMSRules
+from cognite.neat._rules.models.information._rules import InformationRules
+from cognite.neat._rules.models.information._rules_input import InformationInputRules
+from cognite.neat._store import NeatGraphStore
+
+from .exceptions import NeatSessionError
+
+
+@dataclass
+class SessionState:
+    store_type: Literal["memory", "oxigraph"]
+    input_rules: list[ReadRules] = field(default_factory=list)
+    verified_rules: list[VerifiedRules] = field(default_factory=list)
+    _store: NeatGraphStore | None = field(init=False, default=None)
+
+    @property
+    def store(self) -> NeatGraphStore:
+        if not self.has_store:
+            if self.store_type == "oxigraph":
+                self._store = NeatGraphStore.from_oxi_store()
+            else:
+                self._store = NeatGraphStore.from_memory_store()
+        return cast(NeatGraphStore, self._store)
+
+    @property
+    def input_rule(self) -> ReadRules:
+        if not self.input_rules:
+            raise NeatSessionError("No input data model available. Try using [bold].read[/bold] to load a data model.")
+        return self.input_rules[-1]
+
+    @property
+    def information_input_rule(self) -> ReadRules | None:
+        if self.input_rules:
+            for rule in self.input_rules[::-1]:
+                if isinstance(rule.rules, InformationInputRules):
+                    return rule
+        return None
+
+    @property
+    def last_verified_rule(self) -> VerifiedRules:
+        if not self.verified_rules:
+            raise NeatSessionError(
+                "No data model available to verify. Try using  [bold].read[/bold] to load a data model."
+            )
+        return self.verified_rules[-1]
+
+    @property
+    def last_verified_dms_rules(self) -> DMSRules | None:
+        if self.verified_rules:
+            for rules in self.verified_rules[::-1]:
+                if isinstance(rules, DMSRules):
+                    return rules
+        return None
+
+    @property
+    def last_verified_information_rules(self) -> InformationRules | None:
+        if self.verified_rules:
+            for rules in self.verified_rules[::-1]:
+                if isinstance(rules, InformationRules):
+                    return rules
+        return None
+
+    @property
+    def has_store(self) -> bool:
+        return self._store is not None

cognite/neat/_session/_to.py

@@ -0,0 +1,70 @@
+from pathlib import Path
+from typing import Any, overload
+
+from cognite.client import CogniteClient
+
+from cognite.neat._graph import loaders
+from cognite.neat._rules import exporters
+from cognite.neat._session._wizard import space_wizard
+
+from ._state import SessionState
+
+
+class ToAPI:
+    def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+        self.cdf = CDFToAPI(state, client, verbose)
+
+    def excel(
+        self,
+        io: Any,
+    ) -> None:
+        exporter = exporters.ExcelExporter()
+        exporter.export_to_file(self._state.last_verified_rule, Path(io))
+        return None
+
+    @overload
+    def yaml(self, io: None) -> str: ...
+
+    @overload
+    def yaml(self, io: Any) -> None: ...
+
+    def yaml(self, io: Any | None = None) -> str | None:
+        exporter = exporters.YAMLExporter()
+        if io is None:
+            return exporter.export(self._state.last_verified_rule)
+
+        exporter.export_to_file(self._state.last_verified_rule, Path(io))
+        return None
+
+
+class CDFToAPI:
+    def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
+        self._client = client
+        self._state = state
+        self._verbose = verbose
+
+    def instances(self, space: str | None = None):
+        if not self._state.last_verified_dms_rules:
+            raise ValueError("No verified DMS data model available")
+
+        loader = loaders.DMSLoader.from_rules(
+            self._state.last_verified_dms_rules, self._state.store, space_wizard(space=space)
+        )
+
+        if not self._client:
+            raise ValueError("No client provided!")
+
+        return loader.load_into_cdf(self._client)
+
+    def data_model(self):
+        if not self._state.last_verified_dms_rules:
+            raise ValueError("No verified DMS data model available")
+
+        exporter = exporters.DMSExporter()
+
+        if not self._client:
+            raise ValueError("No client provided!")
+
+        return exporter.export_to_cdf(self._state.last_verified_dms_rules, self._client)

cognite/neat/_session/_wizard.py

@@ -0,0 +1,39 @@
+from collections.abc import Sequence
+from typing import Literal, TypeVar, get_args
+
+from rich.prompt import IntPrompt, Prompt
+
+from cognite.neat._rules._constants import PATTERNS
+
+RDFFileType = Literal["Ontology", "IMF Types", "Inference"]
+NeatObjectType = Literal["Data Model", "Instances"]
+
+
+def object_wizard(message: str = "Select object") -> NeatObjectType:
+    return _selection(message, get_args(NeatObjectType))
+
+
+def rdf_dm_wizard(message: str = "Select source:") -> RDFFileType:
+    return _selection(message, get_args(RDFFileType))
+
+
+_T_Option = TypeVar("_T_Option")
+
+
+def _selection(message: str, options: Sequence[_T_Option]) -> _T_Option:
+    option_text = "\n  ".join([f"{i+1}) {option}" for i, option in enumerate(options)])
+    selected_index = (
+        IntPrompt().ask(f"{message}\n  {option_text}\n", choices=list(map(str, range(1, len(options) + 1)))) - 1
+    )
+    return options[selected_index]
+
+
+def space_wizard(message: str = "Set space", space: str | None = None) -> str:
+    while True:
+        user_input = space or Prompt().ask(f"{message}:")
+        if PATTERNS.space_compliance.match(str(user_input)):
+            return user_input
+        else:
+            print(f"Invalid input. Please provide a valid space name. {PATTERNS.space_compliance.pattern}")
+
+        space = ""

cognite/neat/_session/exceptions.py

@@ -0,0 +1,42 @@
+import functools
+from collections.abc import Callable
+from typing import Any
+
+try:
+    from rich import print
+
+    _PREFIX = "[bold red][ERROR][/bold red]"
+except ImportError:
+    _PREFIX = "[ERROR]"
+
+
+class NeatSessionError(Exception):
+    """Base class for all exceptions raised by the NeatSession class."""
+
+    ...
+
+
+def _intercept_session_exceptions(func: Callable):
+    @functools.wraps(func)
+    def wrapper(*args: Any, **kwargs: Any):
+        try:
+            return func(*args, **kwargs)
+        except NeatSessionError as e:
+            action = func.__name__
+            print(f"{_PREFIX} Cannot {action}: {e}")
+
+    return wrapper
+
+
+def intercept_session_exceptions(cls: type):
+    to_check = [cls]
+    while to_check:
+        cls = to_check.pop()
+        for attr_name in dir(cls):
+            if not attr_name.startswith("_"):
+                attr = getattr(cls, attr_name)
+                if callable(attr):
+                    setattr(cls, attr_name, _intercept_session_exceptions(attr))
+                elif isinstance(attr, type):
+                    to_check.append(attr)
+    return cls