cognite-neat 0.107.0__py3-none-any.whl → 0.109.0__py3-none-any.whl

cognite/neat/_rules/transformers/_mapping.py

@@ -13,10 +13,10 @@ from cognite.neat._rules.models.data_types import Enum
 from cognite.neat._rules.models.dms import DMSContainer, DMSEnum, DMSProperty
 from cognite.neat._rules.models.entities import ClassEntity, ContainerEntity, ViewEntity
 
-from ._base import RulesTransformer
+from ._base import VerifiedRulesTransformer
 
 
-class MapOntoTransformers(RulesTransformer[DMSRules, DMSRules], ABC):
+class MapOntoTransformers(VerifiedRulesTransformer[DMSRules, DMSRules], ABC):
     """Base class for transformers that map one rule onto another."""
 
     ...
@@ -100,7 +100,7 @@ class MapOneToOne(MapOntoTransformers):
         return solution
 
 
-class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
+class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
     """Maps properties and classes using the given mapping.
 
     Args:
@@ -232,7 +232,7 @@ class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
         return f"Mapping to {self.mapping.metadata.as_data_model_id()!r}."
 
 
-class AsParentPropertyId(RulesTransformer[DMSRules, DMSRules]):
+class AsParentPropertyId(VerifiedRulesTransformer[DMSRules, DMSRules]):
     """Looks up all view properties that map to the same container property,
     and changes the child view property id to match the parent property id.
     """

cognite/neat/_session/_base.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import Literal
 
 from cognite.client import CogniteClient
@@ -7,14 +8,23 @@ from cognite.neat import _version
 from cognite.neat._client import NeatClient
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import RegexViolationError
+from cognite.neat._issues.errors._general import NeatImportError
 from cognite.neat._rules import importers
-from cognite.neat._rules.models._base_input import InputRules
+from cognite.neat._rules.models import DMSRules
 from cognite.neat._rules.models.information._rules import InformationRules
-from cognite.neat._rules.transformers import ConvertToRules, InformationToDMS, VerifyAnyRules
-from cognite.neat._rules.transformers._converters import ConversionTransformer
+from cognite.neat._rules.transformers import (
+    InformationToDMS,
+    MergeDMSRules,
+    MergeInformationRules,
+    VerifyInformationRules,
+)
+from cognite.neat._store._rules_store import RulesEntity
+from cognite.neat._utils.auxiliary import local_import
 
 from ._collector import _COLLECTOR, Collector
+from ._create import CreateAPI
 from ._drop import DropAPI
+from ._fix import FixAPI
 from ._inspect import InspectAPI
 from ._mapping import MappingAPI
 from ._prepare import PrepareAPI
@@ -24,7 +34,7 @@ from ._show import ShowAPI
 from ._state import SessionState
 from ._to import ToAPI
 from .engine import load_neat_engine
-from .exceptions import NeatSessionError, session_class_wrapper
+from .exceptions import session_class_wrapper
 
 
 @session_class_wrapper
@@ -70,25 +80,42 @@ class NeatSession:
     def __init__(
         self,
         client: CogniteClient | None = None,
-        storage: Literal["memory", "oxigraph"] = "memory",
+        storage: Literal["memory", "oxigraph"] | None = None,
+        storage_path: str | None = None,
         verbose: bool = True,
         load_engine: Literal["newest", "cache", "skip"] = "cache",
     ) -> None:
         self._verbose = verbose
-        self._state = SessionState(store_type=storage, client=NeatClient(client) if client else None)
+        self._state = SessionState(
+            store_type=storage or self._select_most_performant_store(),
+            storage_path=Path(storage_path) if storage_path else None,
+            client=NeatClient(client) if client else None,
+        )
         self.read = ReadAPI(self._state, verbose)
         self.to = ToAPI(self._state, verbose)
+        self.fix = FixAPI(self._state, verbose)
         self.prepare = PrepareAPI(self._state, verbose)
         self.show = ShowAPI(self._state)
         self.set = SetAPI(self._state, verbose)
         self.inspect = InspectAPI(self._state)
         self.mapping = MappingAPI(self._state)
         self.drop = DropAPI(self._state)
+        self.create = CreateAPI(self._state)
         self.opt = OptAPI()
         self.opt._display()
         if load_engine != "skip" and (engine_version := load_neat_engine(client, load_engine)):
             print(f"Neat Engine {engine_version} loaded.")
 
+    def _select_most_performant_store(self) -> Literal["memory", "oxigraph"]:
+        """Select the most performant store based on the current environment."""
+
+        try:
+            local_import("pyoxigraph", "oxi")
+            local_import("oxrdflib", "oxi")
+            return "oxigraph"
+        except NeatImportError:
+            return "memory"
+
     @property
     def version(self) -> str:
         """Get the current version of neat.
@@ -118,46 +145,27 @@ class NeatSession:
             neat.verify()
             ```
         """
-        transformer = VerifyAnyRules(validate=True, client=self._state.client)  # type: ignore[var-annotated]
-        issues = self._state.rule_transform(transformer)
-        if not issues.has_errors:
-            rules = self._state.rule_store.last_verified_rule
-            if isinstance(rules, InformationRules):
-                self._state.instances.store.add_rules(rules)
-
-        if issues:
-            print("You can inspect the issues with the .inspect.issues(...) method.")
-        return issues
+        print("This action has no effect. Neat no longer supports unverified data models.")
+        return IssueList()
 
-    def convert(self, target: Literal["dms", "information"]) -> IssueList:
+    def convert(self, reserved_properties: Literal["error", "warning"] = "warning") -> IssueList:
         """Converts the last verified data model to the target type.
 
         Args:
-            target: The target type to convert the data model to.
+            reserved_properties: What to do with reserved properties. Can be "error" or "warning".
 
         Example:
             Convert to DMS rules
             ```python
-            neat.convert(target="dms")
-            ```
-
-        Example:
-            Convert to Information rules
-            ```python
-            neat.convert(target="information")
+            neat.convert()
             ```
         """
-        converter: ConversionTransformer
-        if target == "dms":
-            converter = InformationToDMS()
-        elif target == "information":
-            converter = ConvertToRules(InformationRules)
-        else:
-            raise NeatSessionError(f"Target {target} not supported.")
+        converter = InformationToDMS(reserved_properties=reserved_properties)
+
         issues = self._state.rule_transform(converter)
 
         if self._verbose and not issues.has_errors:
-            print(f"Rules converted to {target}")
+            print("Rules converted to dms.")
         else:
             print("Conversion failed.")
         if issues:
@@ -174,13 +182,11 @@ class NeatSession:
             "NeatInferredDataModel",
             "v1",
         ),
-        max_number_of_instance: int = 100,
     ) -> IssueList:
         """Data model inference from instances.
 
         Args:
             model_id: The ID of the inferred data model.
-            max_number_of_instance: The maximum number of instances to use for inference.
 
         Example:
             Infer a data model after reading a source file
@@ -191,6 +197,12 @@ class NeatSession:
             neat.infer()
             ```
         """
+        return self._infer_subclasses(model_id)
+
+    def _previous_inference(
+        self, model_id: dm.DataModelId | tuple[str, str, str], max_number_of_instance: int = 100
+    ) -> IssueList:
+        # Temporary keeping the old inference method in case we need to revert back
         model_id = dm.DataModelId.load(model_id)
         importer = importers.InferenceImporter.from_graph_store(
             store=self._state.instances.store,
@@ -199,29 +211,70 @@ class NeatSession:
         )
         return self._state.rule_import(importer)
 
+    def _infer_subclasses(
+        self,
+        model_id: dm.DataModelId | tuple[str, str, str] = (
+            "neat_space",
+            "NeatInferredDataModel",
+            "v1",
+        ),
+    ) -> IssueList:
+        """Infer data model from instances."""
+        last_entity: RulesEntity | None = None
+        if self._state.rule_store.provenance:
+            last_entity = self._state.rule_store.provenance[-1].target_entity
+
+        # Note that this importer behaves as a transformer in the rule store when there is an existing rules.
+        # We are essentially transforming the last entity's information rules into a new set of information rules.
+        importer = importers.SubclassInferenceImporter(
+            issue_list=IssueList(),
+            graph=self._state.instances.store.graph(),
+            rules=last_entity.information if last_entity is not None else None,
+            data_model_id=dm.DataModelId.load(model_id) if last_entity is None else None,
+        )
+
+        def action() -> tuple[InformationRules, DMSRules | None]:
+            unverified_information = importer.to_rules()
+            extra_info = VerifyInformationRules().transform(unverified_information)
+            if not last_entity:
+                return extra_info, None
+            merged_info = MergeInformationRules(extra_info).transform(last_entity.information)
+            if not last_entity.dms:
+                return merged_info, None
+            extra_dms = InformationToDMS(reserved_properties="warning").transform(extra_info)
+            merged_dms = MergeDMSRules(extra_dms).transform(last_entity.dms)
+            return merged_info, merged_dms
+
+        return self._state.rule_store.do_activity(action, importer)
+
     def _repr_html_(self) -> str:
         state = self._state
-        if (
-            not state.instances.has_store
-            and not state.rule_store.has_unverified_rules
-            and not state.rule_store.has_verified_rules
-        ):
+        if state.instances.empty and state.rule_store.empty:
             return "<strong>Empty session</strong>. Get started by reading something with the <em>.read</em> attribute."
 
         output = []
 
-        if state.rule_store.has_unverified_rules and not state.rule_store.has_verified_rules:
-            rules: InputRules = state.rule_store.last_unverified_rule
-            output.append(f"<H2>Unverified Data Model</H2><br />{rules._repr_html_()}")  # type: ignore
+        if state.rule_store.provenance:
+            last_entity = state.rule_store.provenance[-1].target_entity
+            if last_entity.dms:
+                html = last_entity.dms._repr_html_()
+            else:
+                html = last_entity.information._repr_html_()
+            output.append(f"<H2>Data Model</H2><br />{html}")  # type: ignore
 
-        if state.rule_store.has_verified_rules:
-            output.append(f"<H2>Verified Data Model</H2><br />{state.rule_store.last_verified_rule._repr_html_()}")  # type: ignore
-
-        if state.instances.has_store:
+        if not state.instances.empty:
             output.append(f"<H2>Instances</H2> {state.instances.store._repr_html_()}")
 
         return "<br />".join(output)
 
+    def close(self) -> None:
+        """Close the session and release resources."""
+        self._state.instances.store.dataset.close()
+
+    def __del__(self) -> None:
+        """Called by garbage collector"""
+        self.close()
+
 
 @session_class_wrapper
 class OptAPI:

cognite/neat/_session/_create.py

@@ -0,0 +1,133 @@
+from typing import Literal
+
+from cognite.client.data_classes.data_modeling import DataModelIdentifier
+
+from cognite.neat._issues import IssueList
+from cognite.neat._rules.models.dms import DMSValidation
+from cognite.neat._rules.transformers import (
+    IncludeReferenced,
+    ToDataProductModel,
+    ToEnterpriseModel,
+    ToSolutionModel,
+    VerifiedRulesTransformer,
+)
+
+from ._state import SessionState
+from .exceptions import NeatSessionError, session_class_wrapper
+
+
+@session_class_wrapper
+class CreateAPI:
+    """
+    Create new data model based on the given data.
+    """
+
+    def __init__(self, state: SessionState):
+        self._state = state
+
+    def enterprise_model(
+        self,
+        data_model_id: DataModelIdentifier,
+        org_name: str = "My",
+        dummy_property: str = "GUID",
+    ) -> IssueList:
+        """Uses the current data model as a basis to create enterprise data model
+
+        Args:
+            data_model_id: The enterprise data model id that is being created
+            org_name: Organization name to use for the views in the enterprise data model.
+            dummy_property: The dummy property to use as placeholder for the views in the new data model.
+
+        !!! note "Enterprise Data Model Creation"
+
+            Always create an enterprise data model from a Cognite Data Model as this will
+            assure all the Cognite Data Fusion applications to run smoothly, such as
+                - Search
+                - Atlas AI
+                - ...
+
+        !!! note "Move Connections"
+
+            If you want to move the connections to the new data model, set the move_connections
+            to True. This will move the connections to the new data model and use new model
+            views as the source and target views.
+
+        """
+        return self._state.rule_transform(
+            ToEnterpriseModel(
+                new_model_id=data_model_id,
+                org_name=org_name,
+                dummy_property=dummy_property,
+                move_connections=True,
+            )
+        )
+
+    def solution_model(
+        self,
+        data_model_id: DataModelIdentifier,
+        direct_property: str = "enterprise",
+        view_prefix: str = "Enterprise",
+    ) -> IssueList:
+        """Uses the current data model as a basis to create solution data model
+
+        Args:
+            data_model_id: The solution data model id that is being created.
+            direct_property: The property to use for the direct connection between the views in the solution data model
+                and the enterprise data model.
+            view_prefix: The prefix to use for the views in the enterprise data model.
+
+        !!! note "Solution Data Model Mode"
+
+            The read-only solution model will only be able to read from the existing containers
+            from the enterprise data model, therefore the solution data model will not have
+            containers in the solution data model space. Meaning the solution data model views
+            will be read-only.
+
+            The write mode will have additional containers in the solution data model space,
+            allowing in addition to read through the solution model views, also writing to
+            the containers in the solution data model space.
+
+        """
+        return self._state.rule_transform(
+            ToSolutionModel(
+                new_model_id=data_model_id,
+                properties="connection",
+                direct_property=direct_property,
+                view_prefix=view_prefix,
+            )
+        )
+
+    def data_product_model(
+        self,
+        data_model_id: DataModelIdentifier,
+        include: Literal["same-space", "all"] = "same-space",
+    ) -> None:
+        """Uses the current data model as a basis to create data product data model.
+
+        A data product model is a data model that ONLY maps to containers and do not use implements. This is
+        typically used for defining the data in a data product.
+
+        Args:
+            data_model_id: The data product data model id that is being created.
+            include: The views to include in the data product data model. Can be either "same-space" or "all".
+                If you set same-space, only the properties of the views in the same space as the data model
+                will be included.
+        """
+
+        view_ids, container_ids = DMSValidation(
+            self._state.rule_store.last_verified_dms_rules
+        ).imported_views_and_containers_ids()
+        transformers: list[VerifiedRulesTransformer] = []
+        client = self._state.client
+        if (view_ids or container_ids) and client is None:
+            raise NeatSessionError(
+                "No client provided. You are referencing unknown views and containers in your data model, "
+                "NEAT needs a client to lookup the definitions. "
+                "Please set the client in the session, NeatSession(client=client)."
+            )
+        elif (view_ids or container_ids) and client:
+            transformers.append(IncludeReferenced(client, include_properties=True))
+
+        transformers.append(ToDataProductModel(new_model_id=data_model_id, include=include))
+
+        self._state.rule_transform(*transformers)

cognite/neat/_session/_drop.py

@@ -1,7 +1,16 @@
+import difflib
+from collections.abc import Collection
+from typing import Literal
+
+from cognite.client.utils.useful_types import SequenceNotStr
 from rdflib import URIRef
 
+from cognite.neat._constants import COGNITE_MODELS
+from cognite.neat._issues import IssueList
+from cognite.neat._rules.transformers import DropModelViews
+
 from ._state import SessionState
-from .exceptions import session_class_wrapper
+from .exceptions import NeatSessionError, session_class_wrapper
 
 try:
     from rich import print
@@ -17,6 +26,7 @@ class DropAPI:
 
     def __init__(self, state: SessionState):
         self._state = state
+        self.data_model = DropDataModelAPI(state)
 
     def instances(self, type: str | list[str]) -> None:
         """Drop instances from the session.
@@ -31,7 +41,11 @@ class DropAPI:
             ```
         """
         type_list = type if isinstance(type, list) else [type]
-        uri_type_type = dict((v, k) for k, v in self._state.instances.store.queries.types.items())
+
+        # Temporary solution until we agree on the form of specifying named graphs
+        # it will default to the default named graph
+        named_graph = self._state.instances.store.default_named_graph
+        uri_type_type = dict((v, k) for k, v in self._state.instances.store.queries.types(named_graph).items())
         selected_uri_by_type: dict[URIRef, str] = {}
         for type_item in type_list:
             if type_item not in uri_type_type:
@@ -43,3 +57,47 @@ class DropAPI:
         for type_uri, count in result.items():
             print(f"Dropped {count} instances of type {selected_uri_by_type[type_uri]}")
         return None
+
+
+@session_class_wrapper
+class DropDataModelAPI:
+    def __init__(self, state: SessionState):
+        self._state = state
+
+    def views(
+        self,
+        view_external_id: str | SequenceNotStr[str] | None = None,
+        group: Literal["3D", "Annotation", "BaseViews"]
+        | Collection[Literal["3D", "Annotation", "BaseViews"]]
+        | None = None,
+    ) -> IssueList:
+        """Drops views from the data model.
+        Args:
+            view_external_id: The externalId of the view to drop.
+            group: Only applies to CogniteCore model. This is a shorthand for dropping multiple views at once.
+        """
+        if sum([view_external_id is not None, group is not None]) != 1:
+            raise NeatSessionError("Only one of view_external_id or group can be specified.")
+        last_dms = self._state.rule_store.last_verified_dms_rules
+        if group is not None and last_dms.metadata.as_data_model_id() not in COGNITE_MODELS:
+            raise NeatSessionError("Group can only be specified for CogniteCore models.")
+        if view_external_id is not None:
+            existing_views = {view.view.external_id for view in last_dms.views}
+            requested_views = {view_external_id} if isinstance(view_external_id, str) else set(view_external_id)
+            missing_views = requested_views - existing_views
+            if missing_views:
+                suggestions: list[str] = []
+                for view in missing_views:
+                    suggestion = difflib.get_close_matches(view, existing_views, n=1)
+                    if suggestion:
+                        suggestions.append(f"{view} -> {suggestion[0]}")
+                    else:
+                        suggestions.append(f"{view} -> NOT FOUND")
+                raise NeatSessionError(
+                    f"{len(missing_views)} view(s) not found in the data model.\nDid you mean {', '.join(suggestions)}?"
+                )
+        before = len(last_dms.views)
+        issues = self._state.rule_transform(DropModelViews(view_external_id, group))
+        after = len(self._state.rule_store.last_verified_dms_rules.views)
+        print(f"Dropped {before - after} views.")
+        return issues

cognite/neat/_session/_fix.py

@@ -0,0 +1,28 @@
+from cognite.neat._issues._base import IssueList
+from cognite.neat._rules.transformers import (
+    ToCompliantEntities,
+)
+
+from ._state import SessionState
+from .exceptions import session_class_wrapper
+
+
+@session_class_wrapper
+class FixAPI:
+    """Apply variety of fix methods to data model and isntances"""
+
+    def __init__(self, state: SessionState, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+        self.data_model = DataModelFixAPI(state, verbose)
+
+
+@session_class_wrapper
+class DataModelFixAPI:
+    def __init__(self, state: SessionState, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+
+    def cdf_compliant_external_ids(self) -> IssueList:
+        """Convert (information/logical) data model component external ids to CDF compliant entities."""
+        return self._state.rule_transform(ToCompliantEntities())

cognite/neat/_session/_inspect.py

@@ -48,7 +48,6 @@ class InspectAPI:
         self.issues = InspectIssues(state)
         self.outcome = InspectOutcome(state)
 
-    @property
     def properties(self) -> pd.DataFrame:
         """Returns the properties of the current data model.
 
@@ -59,7 +58,23 @@ class InspectAPI:
             neat.inspect.properties
             ```
         """
-        df = self._state.rule_store.last_verified_rule.properties.to_pandas()
+        if self._state.rule_store.empty:
+            return pd.DataFrame()
+        last_entity = self._state.rule_store.provenance[-1].target_entity
+        if last_entity.dms:
+            df = last_entity.dms.properties.to_pandas()
+        else:
+            df = last_entity.information.properties.to_pandas()
+        df.drop(columns=["neatId"], errors="ignore", inplace=True)
+        return df
+
+    def views(self) -> pd.DataFrame:
+        if self._state.rule_store.empty:
+            return pd.DataFrame()
+        last_entity = self._state.rule_store.provenance[-1].target_entity
+        if last_entity.dms is None:
+            return pd.DataFrame()
+        df = last_entity.dms.views.to_pandas()
         df.drop(columns=["neatId"], errors="ignore", inplace=True)
         return df
 
@@ -91,10 +106,10 @@ class InspectIssues:
         return_dataframe: bool = (False if IN_NOTEBOOK else True),  # type: ignore[assignment]
     ) -> pd.DataFrame | None:
         """Returns the issues of the current data model."""
-        if self._state.rule_store.provenance:
-            issues = self._state.rule_store.last_issues
-        elif self._state.instances.store.provenance:
-            issues = self._state.instances.store.provenance[-1].target_entity.issues
+        issues = self._state.rule_store.last_issues
+        if issues is None and self._state.instances.store.provenance:
+            last_change = self._state.instances.store.provenance[-1]
+            issues = last_change.target_entity.issues
         else:
             self._print("No issues found.")
             return pd.DataFrame() if return_dataframe else None
@@ -231,7 +246,7 @@ class InspectUploadOutcome:
                                     if i < 50:
                                         lines.append(f"  * {v}")
                                     elif i == 50 and total > 50:
-                                        lines.append(f"  * ... {total-50} more")
+                                        lines.append(f"  * ... {total - 50} more")
                                     elif i == 50 and total == 50:
                                         lines.append(f"  * {v}")
                                 else:

cognite/neat/_session/_mapping.py

@@ -1,12 +1,11 @@
 from cognite.neat._issues import IssueList
-from cognite.neat._rules.models import DMSRules
 from cognite.neat._rules.models.mapping import load_classic_to_core_mapping
 from cognite.neat._rules.transformers import (
     AsParentPropertyId,
     ChangeViewPrefix,
     IncludeReferenced,
     RuleMapper,
-    RulesTransformer,
+    VerifiedRulesTransformer,
 )
 
 from ._state import SessionState
@@ -43,15 +42,16 @@ class DataModelMappingAPI:
             neat.mapping.classic_to_core(company_prefix="WindFarmX", use_parent_property_name=True)
             ```
         """
-        rules = self._state.rule_store.get_last_successful_entity().result
-        if not isinstance(rules, DMSRules):
-            # Todo better hint of what you should do.
-            raise NeatSessionError(f"Expected DMSRules, got {type(rules)}")
-
+        if self._state.rule_store.empty:
+            raise NeatSessionError("No rules to map")
+        last_entity = self._state.rule_store.provenance[-1].target_entity
+        if last_entity.dms is None:
+            raise NeatSessionError("Data model not converted to DMS. Try running `neat.convert('dms')` first.")
+        rules = last_entity.dms
         if self._state.client is None:
             raise NeatSessionError("Client is required to map classic to core")
 
-        transformers: list[RulesTransformer] = []
+        transformers: list[VerifiedRulesTransformer] = []
         if company_prefix:
             transformers.append(ChangeViewPrefix("Classic", company_prefix))
         transformers.extend(