cognite-neat 0.107.0__py3-none-any.whl → 0.108.0__py3-none-any.whl

cognite/neat/_session/_base.py

@@ -7,11 +7,21 @@ from cognite.neat import _version
 from cognite.neat._client import NeatClient
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import RegexViolationError
+from cognite.neat._issues.errors._general import NeatImportError
 from cognite.neat._rules import importers
 from cognite.neat._rules.models._base_input import InputRules
 from cognite.neat._rules.models.information._rules import InformationRules
-from cognite.neat._rules.transformers import ConvertToRules, InformationToDMS, VerifyAnyRules
-from cognite.neat._rules.transformers._converters import ConversionTransformer
+from cognite.neat._rules.transformers import (
+    ConversionTransformer,
+    ConvertToRules,
+    InformationToDMS,
+    MergeDMSRules,
+    MergeInformationRules,
+    VerifyAnyRules,
+    VerifyInformationRules,
+)
+from cognite.neat._store._rules_store import ModelEntity
+from cognite.neat._utils.auxiliary import local_import
 
 from ._collector import _COLLECTOR, Collector
 from ._drop import DropAPI
@@ -70,12 +80,15 @@ class NeatSession:
     def __init__(
         self,
         client: CogniteClient | None = None,
-        storage: Literal["memory", "oxigraph"] = "memory",
+        storage: Literal["memory", "oxigraph"] | None = None,
         verbose: bool = True,
         load_engine: Literal["newest", "cache", "skip"] = "cache",
     ) -> None:
         self._verbose = verbose
-        self._state = SessionState(store_type=storage, client=NeatClient(client) if client else None)
+        self._state = SessionState(
+            store_type=storage or self._select_most_performant_store(),
+            client=NeatClient(client) if client else None,
+        )
         self.read = ReadAPI(self._state, verbose)
         self.to = ToAPI(self._state, verbose)
         self.prepare = PrepareAPI(self._state, verbose)
@@ -89,6 +102,16 @@ class NeatSession:
         if load_engine != "skip" and (engine_version := load_neat_engine(client, load_engine)):
             print(f"Neat Engine {engine_version} loaded.")
 
+    def _select_most_performant_store(self) -> Literal["memory", "oxigraph"]:
+        """Select the most performant store based on the current environment."""
+
+        try:
+            local_import("pyoxigraph", "oxi")
+            local_import("oxrdflib", "oxi")
+            return "oxigraph"
+        except NeatImportError:
+            return "memory"
+
     @property
     def version(self) -> str:
         """Get the current version of neat.
@@ -199,6 +222,34 @@ class NeatSession:
         )
         return self._state.rule_import(importer)
 
+    def _infer_subclasses(self) -> IssueList:
+        """Infer the subclass of instances."""
+        last_information = self._state.rule_store.last_verified_information_rules
+        issue_list = IssueList()
+        importer = importers.SubclassInferenceImporter(
+            issue_list=issue_list,
+            graph=self._state.instances.store.graph(),
+            rules=last_information,
+            max_number_of_instance=-1,
+        )
+
+        unverified_information = importer.to_rules()
+        verified_information = VerifyInformationRules().transform(unverified_information)
+
+        # Hack into the last information rules to merge the rules with the last verified information rules.
+        # This is to be able to populate the instances store with the inferred subclasses.
+        provenance = self._state.rule_store.provenance
+        for change in reversed(provenance):
+            target_entity = change.target_entity
+            if isinstance(target_entity, ModelEntity) and isinstance(target_entity.result, InformationRules):
+                last_information_rules = change.target_entity.result
+                new_information_rules = MergeInformationRules(verified_information).transform(last_information_rules)
+                object.__setattr__(change.target_entity, "result", new_information_rules)
+                break
+
+        dms_rules = InformationToDMS(reserved_properties="skip").transform(verified_information)
+        return self._state.rule_transform(MergeDMSRules(dms_rules))
+
     def _repr_html_(self) -> str:
         state = self._state
         if (

cognite/neat/_session/_drop.py

@@ -31,7 +31,11 @@ class DropAPI:
             ```
         """
         type_list = type if isinstance(type, list) else [type]
-        uri_type_type = dict((v, k) for k, v in self._state.instances.store.queries.types.items())
+
+        # Temporary solution until we agree on the form of specifying named graphs
+        # it will default to the default named graph
+        named_graph = self._state.instances.store.default_named_graph
+        uri_type_type = dict((v, k) for k, v in self._state.instances.store.queries.types(named_graph).items())
         selected_uri_by_type: dict[URIRef, str] = {}
         for type_item in type_list:
             if type_item not in uri_type_type:

cognite/neat/_session/_inspect.py

@@ -94,7 +94,8 @@ class InspectIssues:
         if self._state.rule_store.provenance:
             issues = self._state.rule_store.last_issues
         elif self._state.instances.store.provenance:
-            issues = self._state.instances.store.provenance[-1].target_entity.issues
+            last_change = self._state.instances.store.provenance[-1]
+            issues = last_change.target_entity.issues
         else:
             self._print("No issues found.")
             return pd.DataFrame() if return_dataframe else None
@@ -231,7 +232,7 @@ class InspectUploadOutcome:
                                     if i < 50:
                                         lines.append(f"  * {v}")
                                     elif i == 50 and total > 50:
-                                        lines.append(f"  * ... {total-50} more")
+                                        lines.append(f"  * ... {total - 50} more")
                                     elif i == 50 and total == 50:
                                         lines.append(f"  * {v}")
                                 else:

cognite/neat/_session/_read.py

@@ -7,7 +7,7 @@ from cognite.neat._client import NeatClient
 from cognite.neat._constants import CLASSIC_CDF_NAMESPACE
 from cognite.neat._graph import examples as instances_examples
 from cognite.neat._graph import extractors
-from cognite.neat._graph.transformers import ConvertLiteral, LiteralToEntity, LookupRelationshipSourceTarget
+from cognite.neat._graph.transformers import ConvertLiteral, LiteralToEntity
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._issues.warnings import MissingCogniteClientWarning
@@ -99,20 +99,41 @@ class CDFReadAPI(BaseReadAPI):
         return self._state.rule_import(importer)
 
     def graph(
-        self, data_model_id: DataModelIdentifier, instance_space: str | SequenceNotStr[str] | None = None
+        self,
+        data_model_id: DataModelIdentifier,
+        instance_space: str | SequenceNotStr[str] | None = None,
+        skip_cognite_views: bool = True,
     ) -> IssueList:
         """Reads a knowledge graph from Cognite Data Fusion (CDF).
 
         Args:
             data_model_id: Tuple of strings with the id of a CDF Data Model.
             instance_space: The instance spaces to extract. If None, all instance spaces are extracted.
+            skip_cognite_views: If True, all Cognite Views are skipped. For example, if you have the CogniteAsset
+                view in you data model, it will ont be used to extract instances.
 
         Returns:
             IssueList: A list of issues that occurred during the extraction.
 
         """
+        return self._graph(data_model_id, instance_space, skip_cognite_views, unpack_json=False)
+
+    def _graph(
+        self,
+        data_model_id: DataModelIdentifier,
+        instance_space: str | SequenceNotStr[str] | None = None,
+        skip_cognite_views: bool = True,
+        unpack_json: bool = False,
+        str_to_ideal_type: bool = False,
+    ) -> IssueList:
         extractor = extractors.DMSGraphExtractor.from_data_model_id(
-            data_model_id, self._get_client, instance_space=instance_space
+            # We are skipping the Cognite Views
+            data_model_id,
+            self._get_client,
+            instance_space=instance_space,
+            skip_cognite_views=skip_cognite_views,
+            unpack_json=unpack_json,
+            str_to_ideal_type=str_to_ideal_type,
         )
         return self._state.write_graph(extractor)
 
@@ -130,7 +151,12 @@ class CDFClassicAPI(BaseReadAPI):
             raise ValueError("No client provided. Please provide a client to read a data model.")
         return self._state.client
 
-    def graph(self, root_asset_external_id: str, limit_per_type: int | None = None) -> IssueList:
+    def graph(
+        self,
+        root_asset_external_id: str,
+        limit_per_type: int | None = None,
+        identifier: Literal["id", "externalId"] = "id",
+    ) -> IssueList:
         """Reads the classic knowledge graph from CDF.
 
         The Classic Graph consists of the following core resource type.
@@ -165,6 +191,8 @@ class CDFClassicAPI(BaseReadAPI):
         Args:
             root_asset_external_id: The external id of the root asset
             limit_per_type: The maximum number of nodes to extract per core node type. If None, all nodes are extracted.
+            identifier: The identifier to use for the core nodes. Note selecting "id" can cause issues if the external
+                ID of the core nodes is missing. Default is "id".
 
         Returns:
             IssueList: A list of issues that occurred during the extraction.
@@ -174,6 +202,18 @@ class CDFClassicAPI(BaseReadAPI):
             neat.read.cdf.graph("root_asset_external_id")
             ```
         """
+        return self._graph(
+            root_asset_external_id, limit_per_type, identifier, reference_timeseries=False, reference_files=False
+        )
+
+    def _graph(
+        self,
+        root_asset_external_id: str,
+        limit_per_type: int | None = None,
+        identifier: Literal["id", "externalId"] = "id",
+        reference_timeseries: bool = False,
+        reference_files: bool = False,
+    ) -> IssueList:
         namespace = CLASSIC_CDF_NAMESPACE
         extractor = extractors.ClassicGraphExtractor(
             self._get_client,
@@ -181,14 +221,12 @@ class CDFClassicAPI(BaseReadAPI):
             limit_per_type=limit_per_type,
             namespace=namespace,
             prefix="Classic",
+            identifier=identifier,
         )
-        issues = self._state.write_graph(extractor)
-        issues.action = "Read Classic Graph"
-        if issues:
-            print("Use the .inspect.issues() for more details")
+        extract_issues = self._state.write_graph(extractor)
+        if identifier == "externalId":
+            self._state.quoted_source_identifiers = True
 
-        # Converting the instances from classic to core
-        self._state.instances.store.transform(LookupRelationshipSourceTarget(namespace, "Classic"))
         self._state.instances.store.transform(
             ConvertLiteral(
                 namespace["ClassicTimeSeries"],
@@ -200,11 +238,21 @@ class CDFClassicAPI(BaseReadAPI):
             LiteralToEntity(None, namespace["source"], "ClassicSourceSystem", "name"),
         )
         # Updating the information model.
-        self._state.rule_store.transform(ClassicPrepareCore(namespace))
+        prepare_issues = self._state.rule_store.transform(
+            ClassicPrepareCore(namespace, reference_timeseries, reference_files)
+        )
         # Update the instance store with the latest rules
         information_rules = self._state.rule_store.last_verified_information_rules
-        self._state.instances.store.rules = information_rules
-        return issues
+        self._state.instances.store.rules[self._state.instances.store.default_named_graph] = information_rules
+
+        all_issues = IssueList(extract_issues + prepare_issues)
+        # Update the provenance with all issue.
+        object.__setattr__(self._state.instances.store.provenance[-1].target_entity, "issues", all_issues)
+        all_issues.action = "Read Classic Graph"
+        if all_issues:
+            print("Use the .inspect.issues() for more details")
+
+        return all_issues
 
 
 @session_class_wrapper
@@ -241,7 +289,6 @@ class ExcelReadAPI(BaseReadAPI):
 class ExcelExampleAPI(BaseReadAPI):
     """Used as example for reading some data model into the NeatSession."""
 
-    @property
     def pump_example(self) -> IssueList:
         """Reads the Hello World pump example into the NeatSession."""
         importer: importers.ExcelImporter = importers.ExcelImporter(catalog.hello_world_pump)

cognite/neat/_session/_set.py

@@ -3,9 +3,12 @@ from cognite.client import data_modeling as dm
 
 from cognite.neat._client import NeatClient
 from cognite.neat._constants import COGNITE_MODELS
+from cognite.neat._graph.transformers import SetNeatType
 from cognite.neat._issues import IssueList
+from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._rules.models import DMSRules
 from cognite.neat._rules.transformers import SetIDDMSModel
+from cognite.neat._utils.text import humanize_collection
 
 from ._state import SessionState
 from .exceptions import NeatSessionError, session_class_wrapper
@@ -44,3 +47,27 @@ class SetAPI:
         if self._verbose:
             print(f"Client set to {self._state.client.config.project} CDF project.")
         return None
+
+    def _instance_sub_type(self, type: str, property: str, drop_property: bool = False) -> None:
+        """Sets the sub type of an instance based on the property."""
+        type_uri = self._state.instances.store.queries.type_uri(type)
+        property_uri = self._state.instances.store.queries.property_uri(property)
+
+        if not type_uri:
+            raise NeatValueError(f"Type {type} does not exist in the graph.")
+        elif len(type_uri) > 1:
+            raise NeatValueError(f"{type} has multiple ids found in the graph: {humanize_collection(type_uri)}.")
+
+        if not property_uri:
+            raise NeatValueError(f"Property {property} does not exist in the graph.")
+        elif len(type_uri) > 1:
+            raise NeatValueError(
+                f"{property} has multiple ids found in the graph: {humanize_collection(property_uri)}."
+            )
+
+        if not self._state.instances.store.queries.type_with_property(type_uri[0], property_uri[0]):
+            raise NeatValueError(f"Property {property} is not defined for type {type}.")
+
+        self._state.instances.store.transform(SetNeatType(type_uri[0], property_uri[0], drop_property))
+
+        return None

cognite/neat/_session/_show.py

@@ -112,7 +112,7 @@ class ShowDataModelAPI(ShowBaseAPI):
         else:
             # This should never happen, but we need to handle it to satisfy mypy
             raise NeatSessionError(
-                f"Unsupported type {type(rules) }. Make sure you have either information or DMS rules."
+                f"Unsupported type {type(rules)}. Make sure you have either information or DMS rules."
             )
         identifier = to_directory_compatible(str(rules.metadata.identifier))
         name = f"{identifier}.html"
@@ -201,7 +201,7 @@ class ShowDataModelImplementsAPI(ShowBaseAPI):
         else:
             # This should never happen, but we need to handle it to satisfy mypy
             raise NeatSessionError(
-                f"Unsupported type {type(rules) }. Make sure you have either information or DMS rules."
+                f"Unsupported type {type(rules)}. Make sure you have either information or DMS rules."
             )
         identifier = to_directory_compatible(str(rules.metadata.identifier))
         name = f"{identifier}_implements.html"
@@ -364,7 +364,7 @@ class ShowInstanceAPI(ShowBaseAPI):
                 'Try setting [bold]NeatSession(storage="oxigraph")[/bold] enable Oxigraph store.'
             )
 
-        if not self._state.instances.store.graph:
+        if not self._state.instances.store.dataset:
             raise NeatSessionError("No instances available. Try using [bold].read[/bold] to load instances.")
 
         di_graph = self._generate_instance_di_graph_and_types()
@@ -395,7 +395,7 @@ class ShowInstanceAPI(ShowBaseAPI):
             object,
             subject_type,
             object_type,
-        ) in self._state.instances.store.graph.query(query):
+        ) in self._state.instances.store.dataset.query(query):
             subject = remove_namespace_from_uri(subject)
             property_ = remove_namespace_from_uri(property_)
             object = remove_namespace_from_uri(object)

cognite/neat/_session/_state.py

@@ -22,11 +22,14 @@ class SessionState:
         self.rule_store = NeatRulesStore()
         self.last_reference: DMSRules | InformationRules | None = None
         self.client = client
+        self.quoted_source_identifiers = False
 
     def rule_transform(self, *transformer: RulesTransformer) -> IssueList:
         if not transformer:
             raise NeatSessionError("No transformers provided.")
         first_transformer = transformer[0]
+
+        # This should not be allowed to be done automatically
         pruned = self.rule_store.prune_until_compatible(first_transformer)
         if pruned:
             type_hint = first_transformer.transform_type_hint()
@@ -64,10 +67,11 @@ class SessionState:
         return issues
 
     def write_graph(self, extractor: KnowledgeGraphExtractor) -> IssueList:
-        self.instances.store.write(extractor)
-        issue_list = self.rule_store.import_graph(extractor)
+        extract_issues = self.instances.store.write(extractor)
+        issues = self.rule_store.import_graph(extractor)
         self.instances.store.add_rules(self.rule_store.last_verified_information_rules)
-        return issue_list
+        issues.extend(extract_issues)
+        return issues
 
 
 @dataclass
@@ -81,7 +85,7 @@ class InstancesState:
     def store(self) -> NeatGraphStore:
         if not self.has_store:
             if self.store_type == "oxigraph":
-                self._store = NeatGraphStore.from_oxi_store()
+                self._store = NeatGraphStore.from_oxi_local_store()
             else:
                 self._store = NeatGraphStore.from_memory_store()
         return cast(NeatGraphStore, self._store)

cognite/neat/_session/_to.py

@@ -104,7 +104,7 @@ class ToAPI:
 
         with zipfile.ZipFile(filepath, "w") as zip_ref:
             zip_ref.writestr(
-                "neat-session/instances/instances.ttl",
+                "neat-session/instances/instances.trig",
                 self._state.instances.store.serialize(),
             )
 
@@ -220,6 +220,9 @@ class CDFToAPI:
             self._state.instances.store,
             instance_space=space,
             client=client,
+            # In case urllib.parse.quote() was run on the extraction, we need to run
+            # urllib.parse.unquote() on the load.
+            unquote_external_ids=self._state.quoted_source_identifiers,
         )
         result = loader.load_into_cdf(client)
         self._state.instances.outcome.append(result)

cognite/neat/_session/_wizard.py

@@ -26,7 +26,7 @@ _T_Option = TypeVar("_T_Option")
 
 
 def _selection(message: str, options: Sequence[_T_Option]) -> _T_Option:
-    option_text = "\n  ".join([f"{i+1}) {option}" for i, option in enumerate(options)])
+    option_text = "\n  ".join([f"{i + 1}) {option}" for i, option in enumerate(options)])
     selected_index = (
         IntPrompt().ask(f"{message}\n  {option_text}\n", choices=list(map(str, range(1, len(options) + 1)))) - 1
     )

cognite/neat/_session/exceptions.py

@@ -3,6 +3,7 @@ from collections.abc import Callable
 from typing import Any
 
 from cognite.neat._issues.errors import CDFMissingClientError, NeatImportError
+from cognite.neat._issues.errors._general import NeatValueError
 
 from ._collector import _COLLECTOR
 
@@ -33,7 +34,7 @@ def _session_method_wrapper(func: Callable, cls_name: str):
         except NeatSessionError as e:
             action = _get_action()
             print(f"{_PREFIX} Cannot {action}: {e}")
-        except (CDFMissingClientError, NeatImportError) as e:
+        except (CDFMissingClientError, NeatImportError, NeatValueError) as e:
             print(f"{_PREFIX} {escape(e.as_message())}")
         except ModuleNotFoundError as e:
             if e.name == "neatengine":