cognite-neat 0.108.0__py3-none-any.whl → 0.109.1__py3-none-any.whl

cognite/neat/_session/_prepare.py

@@ -1,36 +1,19 @@
-from collections.abc import Callable, Collection
-from typing import Any, Literal, cast
+from collections.abc import Callable
+from typing import Any
 
-from cognite.client.data_classes.data_modeling import DataModelIdentifier
 from rdflib import URIRef
 
-from cognite.neat._constants import (
-    get_default_prefixes_and_namespaces,
-)
 from cognite.neat._graph.transformers import (
-    AttachPropertyFromTargetToSource,
     ConnectionToLiteral,
     ConvertLiteral,
     LiteralToEntity,
-    PruneDeadEndEdges,
-    PruneInstancesOfUnknownType,
-    PruneTypes,
     RelationshipAsEdgeTransformer,
-    Transformers,
 )
 from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import NeatValueError
-from cognite.neat._rules.models.dms import DMSValidation
 from cognite.neat._rules.transformers import (
-    IncludeReferenced,
     PrefixEntities,
-    ReduceCogniteModel,
-    RulesTransformer,
-    ToCompliantEntities,
-    ToDataProductModel,
-    ToEnterpriseModel,
-    ToSolutionModel,
 )
 from cognite.neat._utils.text import humanize_collection
 
@@ -59,91 +42,6 @@ class InstancePrepareAPI:
         self._state = state
         self._verbose = verbose
 
-    def dexpi(self) -> None:
-        """Prepares extracted DEXPI graph for further usage in CDF
-
-        !!! note "This method bundles several graph transformers which"
-            - attach values of generic attributes to nodes
-            - create associations between nodes
-            - remove unused generic attributes
-            - remove associations between nodes that do not exist in the extracted graph
-            - remove edges to nodes that do not exist in the extracted graph
-
-        and therefore safeguard CDF from a bad graph
-
-        Example:
-            Apply Dexpi specific transformations:
-            ```python
-            neat.prepare.instances.dexpi()
-            ```
-        """
-
-        DEXPI = get_default_prefixes_and_namespaces()["dexpi"]
-
-        transformers = [
-            # Remove any instance which type is unknown
-            PruneInstancesOfUnknownType(),
-            # Directly connect generic attributes
-            AttachPropertyFromTargetToSource(
-                target_property=DEXPI.Value,
-                target_property_holding_new_property=DEXPI.Name,
-                target_node_type=DEXPI.GenericAttribute,
-                delete_target_node=True,
-            ),
-            # Directly connect associations
-            AttachPropertyFromTargetToSource(
-                target_property=DEXPI.ItemID,
-                target_property_holding_new_property=DEXPI.Type,
-                target_node_type=DEXPI.Association,
-                delete_target_node=True,
-            ),
-            # Remove unused generic attributes and associations
-            PruneTypes([DEXPI.GenericAttribute, DEXPI.Association]),
-            # Remove edges to nodes that do not exist in the extracted graph
-            PruneDeadEndEdges(),
-        ]
-
-        for transformer in transformers:
-            self._state.instances.store.transform(cast(Transformers, transformer))
-
-    def aml(self) -> None:
-        """Prepares extracted AutomationML graph for further usage in CDF
-
-        !!! note "This method bundles several graph transformers which"
-            - attach values of attributes to nodes
-            - remove unused attributes
-            - remove edges to nodes that do not exist in the extracted graph
-
-        and therefore safeguard CDF from a bad graph
-
-        Example:
-            Apply AML specific transformations:
-            ```python
-            neat.prepare.instances.aml()
-            ```
-        """
-
-        AML = get_default_prefixes_and_namespaces()["aml"]
-
-        transformers = [
-            # Remove any instance which type is unknown
-            PruneInstancesOfUnknownType(),
-            # Directly connect generic attributes
-            AttachPropertyFromTargetToSource(
-                target_property=AML.Value,
-                target_property_holding_new_property=AML.Name,
-                target_node_type=AML.Attribute,
-                delete_target_node=True,
-            ),
-            # Prune unused attributes
-            PruneTypes([AML.Attribute]),
-            # # Remove edges to nodes that do not exist in the extracted graph
-            PruneDeadEndEdges(),
-        ]
-
-        for transformer in transformers:
-            self._state.instances.store.transform(cast(Transformers, transformer))
-
     def make_connection_on_exact_match(
         self,
         source: tuple[str, str],
@@ -357,10 +255,6 @@ class DataModelPrepareAPI:
         self._state = state
         self._verbose = verbose
 
-    def cdf_compliant_external_ids(self) -> IssueList:
-        """Convert data model component external ids to CDF compliant entities."""
-        return self._state.rule_transform(ToCompliantEntities())
-
     def prefix(self, prefix: str) -> IssueList:
         """Prefix all views in the data model with the given prefix.
 
@@ -368,143 +262,5 @@ class DataModelPrepareAPI:
             prefix: The prefix to add to the views in the data model.
 
         """
-        return self._state.rule_transform(PrefixEntities(prefix))
-
-    def to_enterprise(
-        self,
-        data_model_id: DataModelIdentifier,
-        org_name: str = "My",
-        dummy_property: str = "GUID",
-        move_connections: bool = False,
-    ) -> IssueList:
-        """Uses the current data model as a basis to create enterprise data model
-
-        Args:
-            data_model_id: The enterprise data model id that is being created
-            org_name: Organization name to use for the views in the enterprise data model.
-            dummy_property: The dummy property to use as placeholder for the views in the new data model.
-            move_connections: If True, the connections will be moved to the new data model.
-
-        !!! note "Enterprise Data Model Creation"
-
-            Always create an enterprise data model from a Cognite Data Model as this will
-            assure all the Cognite Data Fusion applications to run smoothly, such as
-                - Search
-                - Atlas AI
-                - ...
-
-        !!! note "Move Connections"
-
-            If you want to move the connections to the new data model, set the move_connections
-            to True. This will move the connections to the new data model and use new model
-            views as the source and target views.
-
-        """
-        return self._state.rule_transform(
-            ToEnterpriseModel(
-                new_model_id=data_model_id,
-                org_name=org_name,
-                dummy_property=dummy_property,
-                move_connections=move_connections,
-            )
-        )
 
-    def to_solution(
-        self,
-        data_model_id: DataModelIdentifier,
-        org_name: str = "My",
-        mode: Literal["read", "write"] = "read",
-        dummy_property: str = "GUID",
-    ) -> IssueList:
-        """Uses the current data model as a basis to create solution data model
-
-        Args:
-            data_model_id: The solution data model id that is being created.
-            org_name: Organization name to use for the views in the new data model.
-            mode: The mode of the solution data model. Can be either "read" or "write".
-            dummy_property: The dummy property to use as placeholder for the views in the new data model.
-
-        !!! note "Solution Data Model Mode"
-
-            The read-only solution model will only be able to read from the existing containers
-            from the enterprise data model, therefore the solution data model will not have
-            containers in the solution data model space. Meaning the solution data model views
-            will be read-only.
-
-            The write mode will have additional containers in the solution data model space,
-            allowing in addition to reading through the solution model views, also writing to
-            the containers in the solution data model space.
-
-        """
-        return self._state.rule_transform(
-            ToSolutionModel(
-                new_model_id=data_model_id,
-                org_name=org_name,
-                mode=mode,
-                dummy_property=dummy_property,
-            )
-        )
-
-    def to_data_product(
-        self,
-        data_model_id: DataModelIdentifier,
-        org_name: str = "",
-        include: Literal["same-space", "all"] = "same-space",
-    ) -> None:
-        """Uses the current data model as a basis to create data product data model.
-
-        A data product model is a data model that ONLY maps to containers and do not use implements. This is
-        typically used for defining the data in a data product.
-
-        Args:
-            data_model_id: The data product data model id that is being created.
-            org_name: Organization name used as prefix if the model is building on top of a Cognite Data Model.
-            include: The views to include in the data product data model. Can be either "same-space" or "all".
-                If you set same-space, only the properties of the views in the same space as the data model
-                will be included.
-        """
-
-        view_ids, container_ids = DMSValidation(
-            self._state.rule_store.last_verified_dms_rules
-        ).imported_views_and_containers_ids()
-        transformers: list[RulesTransformer] = []
-        client = self._state.client
-        if (view_ids or container_ids) and client is None:
-            raise NeatSessionError(
-                "No client provided. You are referencing unknown views and containers in your data model, "
-                "NEAT needs a client to lookup the definitions. "
-                "Please set the client in the session, NeatSession(client=client)."
-            )
-        elif (view_ids or container_ids) and client:
-            transformers.append(IncludeReferenced(client, include_properties=True))
-
-        transformers.append(
-            ToDataProductModel(
-                new_model_id=data_model_id,
-                org_name=org_name,
-                include=include,
-            )
-        )
-
-        self._state.rule_transform(*transformers)
-
-    def reduce(self, drop: Collection[Literal["3D", "Annotation", "BaseViews"] | str]) -> IssueList:
-        """This is a special method that allow you to drop parts of the data model.
-        This only applies to Cognite Data Models.
-
-        Args:
-            drop: What to drop from the data model. The values 3D, Annotation, and BaseViews are special values that
-                drops multiple views at once. You can also pass externalIds of views to drop individual views.
-
-        """
-        return self._state.rule_transform(ReduceCogniteModel(drop))
-
-    def include_referenced(self) -> IssueList:
-        """Include referenced views and containers in the data model."""
-        if self._state.client is None:
-            raise NeatSessionError(
-                "No client provided. You are referencing unknown views and containers in your data model, "
-                "NEAT needs a client to lookup the definitions. "
-                "Please set the client in the session, NeatSession(client=client)."
-            )
-        return self._state.rule_transform(IncludeReferenced(self._state.client))
+        return self._state.rule_transform(PrefixEntities(prefix))  # type: ignore[arg-type]

cognite/neat/_session/_read.py

@@ -4,10 +4,23 @@ from cognite.client.data_classes.data_modeling import DataModelId, DataModelIden
 from cognite.client.utils.useful_types import SequenceNotStr
 
 from cognite.neat._client import NeatClient
-from cognite.neat._constants import CLASSIC_CDF_NAMESPACE
+from cognite.neat._constants import (
+    CLASSIC_CDF_NAMESPACE,
+    get_default_prefixes_and_namespaces,
+)
 from cognite.neat._graph import examples as instances_examples
 from cognite.neat._graph import extractors
-from cognite.neat._graph.transformers import ConvertLiteral, LiteralToEntity
+from cognite.neat._graph.transformers import (
+    ConvertLiteral,
+    LiteralToEntity,
+    Transformers,
+)
+from cognite.neat._graph.transformers._prune_graph import (
+    AttachPropertyFromTargetToSource,
+    PruneDeadEndEdges,
+    PruneInstancesOfUnknownType,
+    PruneTypes,
+)
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._issues.warnings import MissingCogniteClientWarning
@@ -387,7 +400,7 @@ class XMLReadAPI(BaseReadAPI):
             raise NeatValueError("Only support XML files of DEXPI format at the moment.")
 
     def dexpi(self, io: Any) -> None:
-        """Reads a DEXPI file into the NeatSession.
+        """Reads a DEXPI file into the NeatSession and executes set of predefined transformations.
 
         Args:
             io: file path or url to the DEXPI file
@@ -396,6 +409,13 @@ class XMLReadAPI(BaseReadAPI):
             ```python
             neat.read.xml.dexpi("url_or_path_to_dexpi_file")
             ```
+
+        !!! note "This method bundles several graph transformers which"
+            - attach values of generic attributes to nodes
+            - create associations between nodes
+            - remove unused generic attributes
+            - remove associations between nodes that do not exist in the extracted graph
+            - remove edges to nodes that do not exist in the extracted graph
         """
         path = NeatReader.create(io).materialize_path()
         engine = import_engine()
@@ -404,8 +424,36 @@ class XMLReadAPI(BaseReadAPI):
         extractor = engine.create_extractor()
         self._state.instances.store.write(extractor)
 
+        DEXPI = get_default_prefixes_and_namespaces()["dexpi"]
+
+        transformers = [
+            # Remove any instance which type is unknown
+            PruneInstancesOfUnknownType(),
+            # Directly connect generic attributes
+            AttachPropertyFromTargetToSource(
+                target_property=DEXPI.Value,
+                target_property_holding_new_property=DEXPI.Name,
+                target_node_type=DEXPI.GenericAttribute,
+                delete_target_node=True,
+            ),
+            # Directly connect associations
+            AttachPropertyFromTargetToSource(
+                target_property=DEXPI.ItemID,
+                target_property_holding_new_property=DEXPI.Type,
+                target_node_type=DEXPI.Association,
+                delete_target_node=True,
+            ),
+            # Remove unused generic attributes and associations
+            PruneTypes([DEXPI.GenericAttribute, DEXPI.Association]),
+            # Remove edges to nodes that do not exist in the extracted graph
+            PruneDeadEndEdges(),
+        ]
+
+        for transformer in transformers:
+            self._state.instances.store.transform(cast(Transformers, transformer))
+
     def aml(self, io: Any):
-        """Reads an AML file into NeatSession.
+        """Reads an AML file into NeatSession and executes a set of predefined transformations.
 
         Args:
             io: file path or url to the AML file
@@ -414,6 +462,11 @@ class XMLReadAPI(BaseReadAPI):
             ```python
             neat.read.xml.aml("url_or_path_to_aml_file")
             ```
+
+        !!! note "This method bundles several graph transformers which"
+            - attach values of attributes to nodes
+            - remove unused attributes
+            - remove edges to nodes that do not exist in the extracted graph
         """
         path = NeatReader.create(io).materialize_path()
         engine = import_engine()
@@ -422,6 +475,27 @@ class XMLReadAPI(BaseReadAPI):
         extractor = engine.create_extractor()
         self._state.instances.store.write(extractor)
 
+        AML = get_default_prefixes_and_namespaces()["aml"]
+
+        transformers = [
+            # Remove any instance which type is unknown
+            PruneInstancesOfUnknownType(),
+            # Directly connect generic attributes
+            AttachPropertyFromTargetToSource(
+                target_property=AML.Value,
+                target_property_holding_new_property=AML.Name,
+                target_node_type=AML.Attribute,
+                delete_target_node=True,
+            ),
+            # Prune unused attributes
+            PruneTypes([AML.Attribute]),
+            # # Remove edges to nodes that do not exist in the extracted graph
+            PruneDeadEndEdges(),
+        ]
+
+        for transformer in transformers:
+            self._state.instances.store.transform(cast(Transformers, transformer))
+
 
 @session_class_wrapper
 class RDFReadAPI(BaseReadAPI):

cognite/neat/_session/_set.py

@@ -3,7 +3,7 @@ from cognite.client import data_modeling as dm
 
 from cognite.neat._client import NeatClient
 from cognite.neat._constants import COGNITE_MODELS
-from cognite.neat._graph.transformers import SetNeatType
+from cognite.neat._graph.transformers import SetType
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._rules.models import DMSRules
@@ -21,6 +21,7 @@ class SetAPI:
     def __init__(self, state: SessionState, verbose: bool) -> None:
         self._state = state
         self._verbose = verbose
+        self.instances = SetInstances(state, verbose)
 
     def data_model_id(self, new_model_id: dm.DataModelId | tuple[str, str, str]) -> IssueList:
         """Sets the data model ID of the latest verified data model. Set the data model id as a tuple of strings
@@ -32,7 +33,9 @@ class SetAPI:
             neat.set.data_model_id(("my_data_model_space", "My_Data_Model", "v1"))
             ```
         """
-        rules = self._state.rule_store.get_last_successful_entity().result
+        if self._state.rule_store.empty:
+            raise NeatSessionError("No rules to set the data model ID.")
+        rules = self._state.rule_store.provenance[-1].target_entity.dms
         if isinstance(rules, DMSRules):
             if rules.metadata.as_data_model_id() in COGNITE_MODELS:
                 raise NeatSessionError(
@@ -48,26 +51,45 @@ class SetAPI:
             print(f"Client set to {self._state.client.config.project} CDF project.")
         return None
 
-    def _instance_sub_type(self, type: str, property: str, drop_property: bool = False) -> None:
-        """Sets the sub type of an instance based on the property."""
-        type_uri = self._state.instances.store.queries.type_uri(type)
-        property_uri = self._state.instances.store.queries.property_uri(property)
+
+@session_class_wrapper
+class SetInstances:
+    """Used to change instances"""
+
+    def __init__(self, state: SessionState, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+
+    def type_using_property(self, current_type: str, property_type: str, drop_property: bool = True) -> None:
+        """Replaces the type of all instances with the value of a property.
+
+        Example:
+            All Assets have a property `assetCategory` that we want to use as the type of all asset instances.
+
+            ```python
+            neat.set.instances.replace_type("Asset", "assetCategory")
+            ```
+        """
+        type_uri = self._state.instances.store.queries.type_uri(current_type)
+        property_uri = self._state.instances.store.queries.property_uri(property_type)
 
         if not type_uri:
-            raise NeatValueError(f"Type {type} does not exist in the graph.")
+            raise NeatValueError(f"Type {current_type} does not exist in the graph.")
         elif len(type_uri) > 1:
-            raise NeatValueError(f"{type} has multiple ids found in the graph: {humanize_collection(type_uri)}.")
+            raise NeatValueError(
+                f"{current_type} has multiple ids found in the graph: {humanize_collection(type_uri)}."
+            )
 
         if not property_uri:
-            raise NeatValueError(f"Property {property} does not exist in the graph.")
+            raise NeatValueError(f"Property {property_type} does not exist in the graph.")
         elif len(type_uri) > 1:
             raise NeatValueError(
-                f"{property} has multiple ids found in the graph: {humanize_collection(property_uri)}."
+                f"{property_type} has multiple ids found in the graph: {humanize_collection(property_uri)}."
             )
 
         if not self._state.instances.store.queries.type_with_property(type_uri[0], property_uri[0]):
-            raise NeatValueError(f"Property {property} is not defined for type {type}.")
+            raise NeatValueError(f"Property {property_type} is not defined for type {current_type}.")
 
-        self._state.instances.store.transform(SetNeatType(type_uri[0], property_uri[0], drop_property))
+        self._state.instances.store.transform(SetType(type_uri[0], property_uri[0], drop_property))
 
         return None

cognite/neat/_session/_show.py

@@ -98,22 +98,15 @@ class ShowDataModelAPI(ShowBaseAPI):
         self.implements = ShowDataModelImplementsAPI(self._state)
 
     def __call__(self) -> Any:
-        if not self._state.rule_store.has_verified_rules:
-            raise NeatSessionError(
-                "No verified data model available. Try using [bold].verify()[/bold] to verify data model."
-            )
-
-        rules = self._state.rule_store.last_verified_rule
+        if self._state.rule_store.empty:
+            raise NeatSessionError("No data model available. Try using [bold].read[/bold] to read a new data model.")
+        last_target = self._state.rule_store.provenance[-1].target_entity
+        rules = last_target.dms or last_target.information
 
-        if isinstance(rules, DMSRules):
-            di_graph = self._generate_dms_di_graph(rules)
-        elif isinstance(rules, InformationRules):
-            di_graph = self._generate_info_di_graph(rules)
+        if last_target.dms is not None:
+            di_graph = self._generate_dms_di_graph(last_target.dms)
         else:
-            # This should never happen, but we need to handle it to satisfy mypy
-            raise NeatSessionError(
-                f"Unsupported type {type(rules)}. Make sure you have either information or DMS rules."
-            )
+            di_graph = self._generate_info_di_graph(last_target.information)
         identifier = to_directory_compatible(str(rules.metadata.identifier))
         name = f"{identifier}.html"
 
@@ -187,22 +180,16 @@ class ShowDataModelImplementsAPI(ShowBaseAPI):
         self._state = state
 
     def __call__(self) -> Any:
-        if not self._state.rule_store.has_verified_rules:
-            raise NeatSessionError(
-                "No verified data model available. Try using [bold].verify()[/bold] to verify data model."
-            )
+        if self._state.rule_store.empty:
+            raise NeatSessionError("No data model available. Try using [bold].read[/bold] to read a data model.")
 
-        rules = self._state.rule_store.last_verified_rule
+        last_target = self._state.rule_store.provenance[-1].target_entity
+        rules = last_target.dms or last_target.information
 
-        if isinstance(rules, DMSRules):
-            di_graph = self._generate_dms_di_graph(rules)
-        elif isinstance(rules, InformationRules):
-            di_graph = self._generate_info_di_graph(rules)
+        if last_target.dms is not None:
+            di_graph = self._generate_dms_di_graph(last_target.dms)
         else:
-            # This should never happen, but we need to handle it to satisfy mypy
-            raise NeatSessionError(
-                f"Unsupported type {type(rules)}. Make sure you have either information or DMS rules."
-            )
+            di_graph = self._generate_info_di_graph(last_target.information)
         identifier = to_directory_compatible(str(rules.metadata.identifier))
         name = f"{identifier}_implements.html"
         return self._generate_visualization(di_graph, name)
@@ -325,20 +312,6 @@ class ShowDataModelProvenanceAPI(ShowBaseAPI):
                 )
                 di_graph.add_edge(source_shorten, export_id, label="exported", color="grey")
 
-        for pruned_lists in self._state.rule_store.pruned_by_source_entity_id.values():
-            for prune_path in pruned_lists:
-                for change in prune_path:
-                    source = uri_display_name(change.source_entity.id_)
-                    target = uri_display_name(change.target_entity.id_)
-                    di_graph.add_node(
-                        target,
-                        label=target,
-                        type="Pruned",
-                        title="Pruned",
-                        color=hex_colored_types["Pruned"],
-                    )
-                    di_graph.add_edge(source, target, label="pruned", color="grey")
-
         return di_graph