cognite-neat 0.92.3__py3-none-any.whl → 0.93.0__py3-none-any.whl

cognite/neat/_session/_prepare.py

@@ -0,0 +1,61 @@
+from collections.abc import Collection
+from typing import Literal, cast
+
+from cognite.client.data_classes.data_modeling import DataModelIdentifier
+
+from cognite.neat._issues._base import IssueList
+from cognite.neat._rules._shared import ReadRules
+from cognite.neat._rules.models.information._rules_input import InformationInputRules
+from cognite.neat._rules.transformers import ReduceCogniteModel, ToCompliantEntities, ToExtension
+
+from ._state import SessionState
+
+
+class PrepareAPI:
+    def __init__(self, state: SessionState, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+        self.data_model = DataModelPrepareAPI(state, verbose)
+
+
+class DataModelPrepareAPI:
+    def __init__(self, state: SessionState, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+
+    def cdf_compliant_external_ids(self) -> None:
+        """Convert data model component external ids to CDF compliant entities."""
+        if input := self._state.information_input_rule:
+            output = ToCompliantEntities().transform(input)
+            self._state.input_rules.append(
+                ReadRules(
+                    rules=cast(InformationInputRules, output.get_rules()),
+                    issues=IssueList(),
+                    read_context={},
+                )
+            )
+
+    def to_extension(self, new_data_model_id: DataModelIdentifier, org_name: str | None = None) -> None:
+        """Uses the current data model as a basis to extend from.
+
+        Args:
+            new_data_model_id: The new data model that is extending the current data model.
+            org_name: Organization name to use for the views in the new data model. This is required if you are
+                creating an extension from a Cognite Data Model.
+
+        """
+        if dms := self._state.last_verified_dms_rules:
+            output = ToExtension(new_data_model_id, org_name).transform(dms)
+            self._state.verified_rules.append(output.rules)
+
+    def reduce(self, drop: Collection[Literal["3D", "Annotation", "BaseViews"]]) -> None:
+        """This is a special method that allow you to drop parts of the data model.
+        This only applies to Cognite Data Models.
+
+        Args:
+            drop: Which parts of the data model to drop.
+
+        """
+        if dms := self._state.last_verified_dms_rules:
+            output = ReduceCogniteModel(drop).transform(dms)
+            self._state.verified_rules.append(output.rules)

cognite/neat/_session/_read.py

@@ -0,0 +1,118 @@
+from pathlib import Path
+from typing import Any
+
+from cognite.client import CogniteClient
+from cognite.client.data_classes.data_modeling import DataModelIdentifier
+
+from cognite.neat._graph import examples as instances_examples
+from cognite.neat._graph import extractors
+from cognite.neat._issues import IssueList
+from cognite.neat._rules import importers
+from cognite.neat._rules._shared import ReadRules
+
+from ._state import SessionState
+from ._wizard import NeatObjectType, RDFFileType, object_wizard, rdf_dm_wizard
+
+
+class ReadAPI:
+    def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+        self.cdf = CDFReadAPI(state, client, verbose)
+        self.rdf = RDFReadAPI(state, client, verbose)
+        self.excel = ExcelReadAPI(state, client, verbose)
+
+
+class BaseReadAPI:
+    def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+        self._client = client
+
+    def _store_rules(self, io: Any, input_rules: ReadRules, source: str) -> None:
+        if input_rules.rules:
+            self._state.input_rules.append(input_rules)
+            if self._verbose:
+                if input_rules.issues.has_errors:
+                    print(f"{source} {type(io)} {io} read failed")
+                else:
+                    print(f"{source} {type(io)} {io} read successfully")
+
+    def _return_filepath(self, io: Any) -> Path:
+        if isinstance(io, str):
+            return Path(io)
+        elif isinstance(io, Path):
+            return io
+        else:
+            raise ValueError(f"Expected str or Path, got {type(io)}")
+
+
+class CDFReadAPI(BaseReadAPI):
+    def data_model(self, data_model_id: DataModelIdentifier) -> IssueList:
+        if self._client is None:
+            raise ValueError("No client provided. Please provide a client to read a data model.")
+
+        importer = importers.DMSImporter.from_data_model_id(self._client, data_model_id)
+        input_rules = importer.to_rules()
+        self._store_rules(data_model_id, input_rules, "CDF")
+        return input_rules.issues
+
+
+class ExcelReadAPI(BaseReadAPI):
+    def __call__(self, io: Any) -> IssueList:
+        filepath = self._return_filepath(io)
+        input_rules: ReadRules = importers.ExcelImporter(filepath).to_rules()
+        self._store_rules(io, input_rules, "Excel")
+        return input_rules.issues
+
+
+class RDFReadAPI(BaseReadAPI):
+    def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
+        super().__init__(state, client, verbose)
+        self.examples = RDFExamples(state)
+
+    def _ontology(self, io: Any) -> IssueList:
+        filepath = self._return_filepath(io)
+        input_rules: ReadRules = importers.OWLImporter.from_file(filepath).to_rules()
+        self._store_rules(io, input_rules, "Ontology")
+        return input_rules.issues
+
+    def _imf(self, io: Any) -> IssueList:
+        filepath = self._return_filepath(io)
+        input_rules: ReadRules = importers.IMFImporter.from_file(filepath).to_rules()
+        self._store_rules(io, input_rules, "IMF Types")
+        return input_rules.issues
+
+    def __call__(
+        self,
+        io: Any,
+        type: NeatObjectType | None = None,
+        source: RDFFileType | None = None,
+    ) -> IssueList:
+        if type is None:
+            type = object_wizard()
+
+        if type.lower() == "Data Model".lower():
+            source = source or rdf_dm_wizard("What type of data model is the RDF?")
+            if source == "Ontology":
+                return self._ontology(io)
+            elif source == "IMF":
+                return self._imf(io)
+            else:
+                raise ValueError(f"Expected ontology, imf or instances, got {source}")
+
+        elif type.lower() == "Instances".lower():
+            self._state.store.write(extractors.RdfFileExtractor(self._return_filepath(io)))
+            return IssueList()
+        else:
+            raise ValueError(f"Expected data model or instances, got {type}")
+
+
+class RDFExamples:
+    def __init__(self, state: SessionState) -> None:
+        self._state = state
+
+    @property
+    def nordic44(self) -> IssueList:
+        self._state.store.write(extractors.RdfFileExtractor(instances_examples.nordic44_knowledge_graph))
+        return IssueList()

cognite/neat/_session/_show.py

@@ -0,0 +1,96 @@
+from typing import Any, cast
+
+import networkx as nx
+from ipycytoscape import CytoscapeWidget  # type: ignore
+from IPython.display import display
+
+from cognite.neat._rules.models.dms._rules import DMSRules
+from cognite.neat._rules.models.entities._single_value import ViewEntity
+
+from ._state import SessionState
+
+
+class ShowAPI:
+    def __init__(self, state: SessionState) -> None:
+        self._state = state
+        self.data_model = ShowDataModelAPI(self._state)
+
+
+class ShowDataModelAPI:
+    def __init__(self, state: SessionState) -> None:
+        self._state = state
+
+    def __call__(self) -> Any:
+        if self._state.last_verified_dms_rules:
+            digraph = self._generate_dms_di_graph()
+            widget = self._generate_widget()
+            widget.graph.add_graph_from_networkx(digraph)
+            return display(widget)
+
+    def _generate_dms_di_graph(self) -> nx.DiGraph:
+        """Generate a DiGraph from the last verified DMS rules."""
+        G = nx.DiGraph()
+
+        nodes, edges = self._generate_dms_rules_nodes_and_edges()
+        G.add_nodes_from(nodes)
+        G.add_edges_from(edges)
+        for node in G.nodes:
+            G.nodes[node]["label"] = node
+
+        return G
+
+    def _generate_dms_rules_nodes_and_edges(self) -> tuple[list[str], list[tuple[str, str, dict]]]:
+        """Generate nodes and edges for the last verified DMS rules for DiGraph."""
+
+        nodes = []
+        edges = []
+
+        for prop_ in cast(DMSRules, self._state.last_verified_dms_rules).properties:
+            nodes.append(prop_.view.suffix)
+
+            if prop_.connection and isinstance(prop_.value_type, ViewEntity):
+                label = f"{prop_.property_} [{0 if prop_.nullable else 1}..{ '' if prop_.is_list else 1}]"
+                edges.append((prop_.view.suffix, prop_.value_type.suffix, {"label": label}))
+
+        for view in cast(DMSRules, self._state.last_verified_dms_rules).views:
+            nodes.append(view.view.suffix)
+
+            if view.implements:
+                for implement in view.implements:
+                    edges.append((view.view.suffix, implement.suffix, {"label": "implements"}))
+
+        return nodes, edges
+
+    def _generate_widget(self):
+        """Generates an empty a CytoscapeWidget."""
+        widget = CytoscapeWidget()
+        widget.layout.height = "500px"
+
+        widget.set_style(
+            [
+                {
+                    "selector": "node",
+                    "css": {
+                        "content": "data(label)",
+                        "text-valign": "center",
+                        "color": "black",
+                        "background-color": "#33C4FF",
+                        "font-size": "10px",
+                        "width": "mapData(score, 0, 1, 10, 50)",
+                        "height": "mapData(score, 0, 1, 10, 50)",
+                    },
+                },
+                {
+                    "selector": "edge",
+                    "style": {
+                        "width": 1,
+                        "target-arrow-shape": "triangle",
+                        "curve-style": "bezier",
+                        "label": "data(label)",
+                        "font-size": "8px",
+                    },
+                },
+            ]
+        )
+
+        return widget

cognite/neat/_session/_state.py

@@ -0,0 +1,69 @@
+from dataclasses import dataclass, field
+from typing import Literal, cast
+
+from cognite.neat._rules._shared import ReadRules, VerifiedRules
+from cognite.neat._rules.models.dms._rules import DMSRules
+from cognite.neat._rules.models.information._rules import InformationRules
+from cognite.neat._rules.models.information._rules_input import InformationInputRules
+from cognite.neat._store import NeatGraphStore
+
+from .exceptions import NeatSessionError
+
+
+@dataclass
+class SessionState:
+    store_type: Literal["memory", "oxigraph"]
+    input_rules: list[ReadRules] = field(default_factory=list)
+    verified_rules: list[VerifiedRules] = field(default_factory=list)
+    _store: NeatGraphStore | None = field(init=False, default=None)
+
+    @property
+    def store(self) -> NeatGraphStore:
+        if not self.has_store:
+            if self.store_type == "oxigraph":
+                self._store = NeatGraphStore.from_oxi_store()
+            else:
+                self._store = NeatGraphStore.from_memory_store()
+        return cast(NeatGraphStore, self._store)
+
+    @property
+    def input_rule(self) -> ReadRules:
+        if not self.input_rules:
+            raise NeatSessionError("No input data model available. Try using [bold].read[/bold] to load a data model.")
+        return self.input_rules[-1]
+
+    @property
+    def information_input_rule(self) -> ReadRules | None:
+        if self.input_rules:
+            for rule in self.input_rules[::-1]:
+                if isinstance(rule.rules, InformationInputRules):
+                    return rule
+        return None
+
+    @property
+    def last_verified_rule(self) -> VerifiedRules:
+        if not self.verified_rules:
+            raise NeatSessionError(
+                "No data model available to verify. Try using  [bold].read[/bold] to load a data model."
+            )
+        return self.verified_rules[-1]
+
+    @property
+    def last_verified_dms_rules(self) -> DMSRules | None:
+        if self.verified_rules:
+            for rules in self.verified_rules[::-1]:
+                if isinstance(rules, DMSRules):
+                    return rules
+        return None
+
+    @property
+    def last_verified_information_rules(self) -> InformationRules | None:
+        if self.verified_rules:
+            for rules in self.verified_rules[::-1]:
+                if isinstance(rules, InformationRules):
+                    return rules
+        return None
+
+    @property
+    def has_store(self) -> bool:
+        return self._store is not None

cognite/neat/_session/_to.py

@@ -0,0 +1,70 @@
+from pathlib import Path
+from typing import Any, overload
+
+from cognite.client import CogniteClient
+
+from cognite.neat._graph import loaders
+from cognite.neat._rules import exporters
+from cognite.neat._session._wizard import space_wizard
+
+from ._state import SessionState
+
+
+class ToAPI:
+    def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+        self.cdf = CDFToAPI(state, client, verbose)
+
+    def excel(
+        self,
+        io: Any,
+    ) -> None:
+        exporter = exporters.ExcelExporter()
+        exporter.export_to_file(self._state.last_verified_rule, Path(io))
+        return None
+
+    @overload
+    def yaml(self, io: None) -> str: ...
+
+    @overload
+    def yaml(self, io: Any) -> None: ...
+
+    def yaml(self, io: Any | None = None) -> str | None:
+        exporter = exporters.YAMLExporter()
+        if io is None:
+            return exporter.export(self._state.last_verified_rule)
+
+        exporter.export_to_file(self._state.last_verified_rule, Path(io))
+        return None
+
+
+class CDFToAPI:
+    def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
+        self._client = client
+        self._state = state
+        self._verbose = verbose
+
+    def instances(self, space: str | None = None):
+        if not self._state.last_verified_dms_rules:
+            raise ValueError("No verified DMS data model available")
+
+        loader = loaders.DMSLoader.from_rules(
+            self._state.last_verified_dms_rules, self._state.store, space_wizard(space=space)
+        )
+
+        if not self._client:
+            raise ValueError("No client provided!")
+
+        return loader.load_into_cdf(self._client)
+
+    def data_model(self):
+        if not self._state.last_verified_dms_rules:
+            raise ValueError("No verified DMS data model available")
+
+        exporter = exporters.DMSExporter()
+
+        if not self._client:
+            raise ValueError("No client provided!")
+
+        return exporter.export_to_cdf(self._state.last_verified_dms_rules, self._client)

cognite/neat/_session/_wizard.py

@@ -0,0 +1,39 @@
+from collections.abc import Sequence
+from typing import Literal, TypeVar, get_args
+
+from rich.prompt import IntPrompt, Prompt
+
+from cognite.neat._rules._constants import PATTERNS
+
+RDFFileType = Literal["Ontology", "IMF Types", "Inference"]
+NeatObjectType = Literal["Data Model", "Instances"]
+
+
+def object_wizard(message: str = "Select object") -> NeatObjectType:
+    return _selection(message, get_args(NeatObjectType))
+
+
+def rdf_dm_wizard(message: str = "Select source:") -> RDFFileType:
+    return _selection(message, get_args(RDFFileType))
+
+
+_T_Option = TypeVar("_T_Option")
+
+
+def _selection(message: str, options: Sequence[_T_Option]) -> _T_Option:
+    option_text = "\n  ".join([f"{i+1}) {option}" for i, option in enumerate(options)])
+    selected_index = (
+        IntPrompt().ask(f"{message}\n  {option_text}\n", choices=list(map(str, range(1, len(options) + 1)))) - 1
+    )
+    return options[selected_index]
+
+
+def space_wizard(message: str = "Set space", space: str | None = None) -> str:
+    while True:
+        user_input = space or Prompt().ask(f"{message}:")
+        if PATTERNS.space_compliance.match(str(user_input)):
+            return user_input
+        else:
+            print(f"Invalid input. Please provide a valid space name. {PATTERNS.space_compliance.pattern}")
+
+        space = ""

cognite/neat/_session/exceptions.py

@@ -0,0 +1,42 @@
+import functools
+from collections.abc import Callable
+from typing import Any
+
+try:
+    from rich import print
+
+    _PREFIX = "[bold red][ERROR][/bold red]"
+except ImportError:
+    _PREFIX = "[ERROR]"
+
+
+class NeatSessionError(Exception):
+    """Base class for all exceptions raised by the NeatSession class."""
+
+    ...
+
+
+def _intercept_session_exceptions(func: Callable):
+    @functools.wraps(func)
+    def wrapper(*args: Any, **kwargs: Any):
+        try:
+            return func(*args, **kwargs)
+        except NeatSessionError as e:
+            action = func.__name__
+            print(f"{_PREFIX} Cannot {action}: {e}")
+
+    return wrapper
+
+
+def intercept_session_exceptions(cls: type):
+    to_check = [cls]
+    while to_check:
+        cls = to_check.pop()
+        for attr_name in dir(cls):
+            if not attr_name.startswith("_"):
+                attr = getattr(cls, attr_name)
+                if callable(attr):
+                    setattr(cls, attr_name, _intercept_session_exceptions(attr))
+                elif isinstance(attr, type):
+                    to_check.append(attr)
+    return cls

cognite/neat/{store → _store}/_base.py

@@ -3,22 +3,22 @@ import warnings
 from collections.abc import Iterable
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import cast
+from typing import Any, cast
 
 import pandas as pd
+from pandas import Index
 from rdflib import Graph, Namespace, URIRef
 from rdflib.plugins.stores.sparqlstore import SPARQLUpdateStore
 
-from cognite.neat.constants import DEFAULT_NAMESPACE
-from cognite.neat.graph._shared import MIMETypes
-from cognite.neat.graph.extractors import RdfFileExtractor, TripleExtractors
-from cognite.neat.graph.models import InstanceType, Triple
-from cognite.neat.graph.queries import Queries
-from cognite.neat.graph.transformers import Transformers
-from cognite.neat.rules.analysis import InformationAnalysis
-from cognite.neat.rules.models import InformationRules
-from cognite.neat.rules.models.entities import ClassEntity
-from cognite.neat.utils.auxiliary import local_import
+from cognite.neat._constants import DEFAULT_NAMESPACE
+from cognite.neat._graph.extractors import RdfFileExtractor, TripleExtractors
+from cognite.neat._graph.models import InstanceType, Triple
+from cognite.neat._graph.queries import Queries
+from cognite.neat._graph.transformers import Transformers
+from cognite.neat._rules.analysis import InformationAnalysis
+from cognite.neat._rules.models import InformationRules
+from cognite.neat._rules.models.entities import ClassEntity
+from cognite.neat._utils.auxiliary import local_import
 
 from ._provenance import Change, Provenance
 
@@ -158,20 +158,29 @@ class NeatGraphStore:
 
     def write(self, extractor: TripleExtractors) -> None:
         _start = datetime.now(timezone.utc)
+        success = True
 
-        if isinstance(extractor, RdfFileExtractor):
-            self._parse_file(extractor.filepath, extractor.mime_type, extractor.base_uri)
+        if isinstance(extractor, RdfFileExtractor) and not extractor.issue_list.has_errors:
+            self._parse_file(extractor.filepath, cast(str, extractor.mime_type), extractor.base_uri)
+        elif isinstance(extractor, RdfFileExtractor):
+            success = False
+            issue_text = "\n".join([issue.as_message() for issue in extractor.issue_list])
+            warnings.warn(
+                f"Cannot write to graph store with {type(extractor).__name__}, errors found in file:\n{issue_text}",
+                stacklevel=2,
+            )
         else:
             self._add_triples(extractor.extract())
 
-        self.provenance.append(
-            Change.record(
-                activity=f"{type(extractor).__name__}",
-                start=_start,
-                end=datetime.now(timezone.utc),
-                description=f"Extracted triples to graph store using {type(extractor).__name__}",
+        if success:
+            self.provenance.append(
+                Change.record(
+                    activity=f"{type(extractor).__name__}",
+                    start=_start,
+                    end=datetime.now(timezone.utc),
+                    description=f"Extracted triples to graph store using {type(extractor).__name__}",
+                )
             )
-        )
 
     def read(self, class_: str) -> Iterable[tuple[str, dict[str | InstanceType, list[str]]]]:
         """Read instances for given view from the graph store."""
@@ -234,7 +243,7 @@ class NeatGraphStore:
     def _parse_file(
         self,
         filepath: Path,
-        mime_type: MIMETypes = "application/rdf+xml",
+        mime_type: str = "application/rdf+xml",
         base_uri: URIRef | None = None,
     ) -> None:
         """Imports graph data from file.
@@ -342,16 +351,38 @@ class NeatGraphStore:
     def _repr_html_(self) -> str:
         provenance = self.provenance._repr_html_()
         summary: pd.DataFrame = self.summary
-        summary_text = (
-            "<br /><strong>Graph is empty</strong><br />"
-            if summary.empty
-            else f"<br /><strong>Graph content</strong><br />{cast(pd.DataFrame, summary)._repr_html_()}"  # type: ignore[operator]
-        )
 
-        return (
-            f"<strong>{type(self).__name__}</strong> A graph store is a container for storing triples. "
-            "It can be queried and transformed to extract information.<br />"
-            "<strong>Provenance</strong> Provenance is a record of changes that have occurred in the graph store.<br />"
-            f"{provenance}"
-            f"{summary_text}"
+        if summary.empty:
+            summary_text = "<br /><strong>Graph is empty</strong><br />"
+        else:
+            summary_text = (
+                "<br /><strong>Overview</strong>:"  # type: ignore
+                f"<ul><li>{len(summary)} types</strong></li>"
+                f"<li>{sum(summary['Occurrence'])} instances</strong></li></ul>"
+                f"{cast(pd.DataFrame, self._shorten_summary(summary))._repr_html_()}"  # type: ignore[operator]
+            )
+
+        return f"{summary_text}" f"{provenance}"
+
+    def _shorten_summary(self, summary: pd.DataFrame) -> pd.DataFrame:
+        """Shorten summary to top 5 types by occurrence."""
+        top_5_rows = summary.head(5)
+        last_row = summary.tail(1)
+
+        indexes = [
+            *top_5_rows.index.tolist(),
+            "...",
+            *last_row.index.tolist(),
+        ]
+
+        shorter_summary = pd.concat(
+            [
+                top_5_rows,
+                pd.DataFrame([["..."] * summary.shape[1]], columns=summary.columns),
+                last_row,
+            ],
+            ignore_index=True,
         )
+        shorter_summary.index = cast(Index[Any], indexes)
+
+        return shorter_summary

cognite/neat/{store → _store}/_provenance.py

@@ -15,8 +15,8 @@ from typing import TypeVar
 
 from rdflib import PROV, RDF, Literal, URIRef
 
+from cognite.neat._constants import DEFAULT_NAMESPACE
 from cognite.neat._shared import FrozenNeatObject, NeatList
-from cognite.neat.constants import DEFAULT_NAMESPACE
 
 
 @dataclass(frozen=True)
@@ -114,3 +114,13 @@ class Provenance(NeatList[Change]):
 
     def __setitem__(self, *args, **kwargs):
         raise TypeError("Cannot modify change from provenance")
+
+    def _repr_html_(self) -> str:
+        text = "<br /><strong>Provenance</strong>:<ul>"
+
+        for change in self:
+            text += f"<li>{change.description}</li>"
+
+        text += "</ul>"
+
+        return text