cognite-neat 0.107.0__py3-none-any.whl → 0.109.0__py3-none-any.whl

cognite/neat/_session/_prepare.py

@@ -1,36 +1,19 @@
-from collections.abc import Callable, Collection
-from typing import Any, Literal, cast
+from collections.abc import Callable
+from typing import Any
 
-from cognite.client.data_classes.data_modeling import DataModelIdentifier
 from rdflib import URIRef
 
-from cognite.neat._constants import (
-    get_default_prefixes_and_namespaces,
-)
 from cognite.neat._graph.transformers import (
-    AttachPropertyFromTargetToSource,
     ConnectionToLiteral,
     ConvertLiteral,
     LiteralToEntity,
-    PruneDeadEndEdges,
-    PruneInstancesOfUnknownType,
-    PruneTypes,
     RelationshipAsEdgeTransformer,
-    Transformers,
 )
 from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import NeatValueError
-from cognite.neat._rules.models.dms import DMSValidation
 from cognite.neat._rules.transformers import (
-    IncludeReferenced,
     PrefixEntities,
-    ReduceCogniteModel,
-    RulesTransformer,
-    ToCompliantEntities,
-    ToDataProductModel,
-    ToEnterpriseModel,
-    ToSolutionModel,
 )
 from cognite.neat._utils.text import humanize_collection
 
@@ -59,91 +42,6 @@ class InstancePrepareAPI:
         self._state = state
         self._verbose = verbose
 
-    def dexpi(self) -> None:
-        """Prepares extracted DEXPI graph for further usage in CDF
-
-        !!! note "This method bundles several graph transformers which"
-            - attach values of generic attributes to nodes
-            - create associations between nodes
-            - remove unused generic attributes
-            - remove associations between nodes that do not exist in the extracted graph
-            - remove edges to nodes that do not exist in the extracted graph
-
-        and therefore safeguard CDF from a bad graph
-
-        Example:
-            Apply Dexpi specific transformations:
-            ```python
-            neat.prepare.instances.dexpi()
-            ```
-        """
-
-        DEXPI = get_default_prefixes_and_namespaces()["dexpi"]
-
-        transformers = [
-            # Remove any instance which type is unknown
-            PruneInstancesOfUnknownType(),
-            # Directly connect generic attributes
-            AttachPropertyFromTargetToSource(
-                target_property=DEXPI.Value,
-                target_property_holding_new_property=DEXPI.Name,
-                target_node_type=DEXPI.GenericAttribute,
-                delete_target_node=True,
-            ),
-            # Directly connect associations
-            AttachPropertyFromTargetToSource(
-                target_property=DEXPI.ItemID,
-                target_property_holding_new_property=DEXPI.Type,
-                target_node_type=DEXPI.Association,
-                delete_target_node=True,
-            ),
-            # Remove unused generic attributes and associations
-            PruneTypes([DEXPI.GenericAttribute, DEXPI.Association]),
-            # Remove edges to nodes that do not exist in the extracted graph
-            PruneDeadEndEdges(),
-        ]
-
-        for transformer in transformers:
-            self._state.instances.store.transform(cast(Transformers, transformer))
-
-    def aml(self) -> None:
-        """Prepares extracted AutomationML graph for further usage in CDF
-
-        !!! note "This method bundles several graph transformers which"
-            - attach values of attributes to nodes
-            - remove unused attributes
-            - remove edges to nodes that do not exist in the extracted graph
-
-        and therefore safeguard CDF from a bad graph
-
-        Example:
-            Apply AML specific transformations:
-            ```python
-            neat.prepare.instances.aml()
-            ```
-        """
-
-        AML = get_default_prefixes_and_namespaces()["aml"]
-
-        transformers = [
-            # Remove any instance which type is unknown
-            PruneInstancesOfUnknownType(),
-            # Directly connect generic attributes
-            AttachPropertyFromTargetToSource(
-                target_property=AML.Value,
-                target_property_holding_new_property=AML.Name,
-                target_node_type=AML.Attribute,
-                delete_target_node=True,
-            ),
-            # Prune unused attributes
-            PruneTypes([AML.Attribute]),
-            # # Remove edges to nodes that do not exist in the extracted graph
-            PruneDeadEndEdges(),
-        ]
-
-        for transformer in transformers:
-            self._state.instances.store.transform(cast(Transformers, transformer))
-
     def make_connection_on_exact_match(
         self,
         source: tuple[str, str],
@@ -357,10 +255,6 @@ class DataModelPrepareAPI:
         self._state = state
         self._verbose = verbose
 
-    def cdf_compliant_external_ids(self) -> IssueList:
-        """Convert data model component external ids to CDF compliant entities."""
-        return self._state.rule_transform(ToCompliantEntities())
-
     def prefix(self, prefix: str) -> IssueList:
         """Prefix all views in the data model with the given prefix.
 
@@ -368,143 +262,5 @@ class DataModelPrepareAPI:
             prefix: The prefix to add to the views in the data model.
 
         """
-        return self._state.rule_transform(PrefixEntities(prefix))
-
-    def to_enterprise(
-        self,
-        data_model_id: DataModelIdentifier,
-        org_name: str = "My",
-        dummy_property: str = "GUID",
-        move_connections: bool = False,
-    ) -> IssueList:
-        """Uses the current data model as a basis to create enterprise data model
-
-        Args:
-            data_model_id: The enterprise data model id that is being created
-            org_name: Organization name to use for the views in the enterprise data model.
-            dummy_property: The dummy property to use as placeholder for the views in the new data model.
-            move_connections: If True, the connections will be moved to the new data model.
-
-        !!! note "Enterprise Data Model Creation"
-
-            Always create an enterprise data model from a Cognite Data Model as this will
-            assure all the Cognite Data Fusion applications to run smoothly, such as
-                - Search
-                - Atlas AI
-                - ...
-
-        !!! note "Move Connections"
-
-            If you want to move the connections to the new data model, set the move_connections
-            to True. This will move the connections to the new data model and use new model
-            views as the source and target views.
-
-        """
-        return self._state.rule_transform(
-            ToEnterpriseModel(
-                new_model_id=data_model_id,
-                org_name=org_name,
-                dummy_property=dummy_property,
-                move_connections=move_connections,
-            )
-        )
 
-    def to_solution(
-        self,
-        data_model_id: DataModelIdentifier,
-        org_name: str = "My",
-        mode: Literal["read", "write"] = "read",
-        dummy_property: str = "GUID",
-    ) -> IssueList:
-        """Uses the current data model as a basis to create solution data model
-
-        Args:
-            data_model_id: The solution data model id that is being created.
-            org_name: Organization name to use for the views in the new data model.
-            mode: The mode of the solution data model. Can be either "read" or "write".
-            dummy_property: The dummy property to use as placeholder for the views in the new data model.
-
-        !!! note "Solution Data Model Mode"
-
-            The read-only solution model will only be able to read from the existing containers
-            from the enterprise data model, therefore the solution data model will not have
-            containers in the solution data model space. Meaning the solution data model views
-            will be read-only.
-
-            The write mode will have additional containers in the solution data model space,
-            allowing in addition to reading through the solution model views, also writing to
-            the containers in the solution data model space.
-
-        """
-        return self._state.rule_transform(
-            ToSolutionModel(
-                new_model_id=data_model_id,
-                org_name=org_name,
-                mode=mode,
-                dummy_property=dummy_property,
-            )
-        )
-
-    def to_data_product(
-        self,
-        data_model_id: DataModelIdentifier,
-        org_name: str = "",
-        include: Literal["same-space", "all"] = "same-space",
-    ) -> None:
-        """Uses the current data model as a basis to create data product data model.
-
-        A data product model is a data model that ONLY maps to containers and do not use implements. This is
-        typically used for defining the data in a data product.
-
-        Args:
-            data_model_id: The data product data model id that is being created.
-            org_name: Organization name used as prefix if the model is building on top of a Cognite Data Model.
-            include: The views to include in the data product data model. Can be either "same-space" or "all".
-                If you set same-space, only the properties of the views in the same space as the data model
-                will be included.
-        """
-
-        view_ids, container_ids = DMSValidation(
-            self._state.rule_store.last_verified_dms_rules
-        ).imported_views_and_containers_ids()
-        transformers: list[RulesTransformer] = []
-        client = self._state.client
-        if (view_ids or container_ids) and client is None:
-            raise NeatSessionError(
-                "No client provided. You are referencing unknown views and containers in your data model, "
-                "NEAT needs a client to lookup the definitions. "
-                "Please set the client in the session, NeatSession(client=client)."
-            )
-        elif (view_ids or container_ids) and client:
-            transformers.append(IncludeReferenced(client, include_properties=True))
-
-        transformers.append(
-            ToDataProductModel(
-                new_model_id=data_model_id,
-                org_name=org_name,
-                include=include,
-            )
-        )
-
-        self._state.rule_transform(*transformers)
-
-    def reduce(self, drop: Collection[Literal["3D", "Annotation", "BaseViews"] | str]) -> IssueList:
-        """This is a special method that allow you to drop parts of the data model.
-        This only applies to Cognite Data Models.
-
-        Args:
-            drop: What to drop from the data model. The values 3D, Annotation, and BaseViews are special values that
-                drops multiple views at once. You can also pass externalIds of views to drop individual views.
-
-        """
-        return self._state.rule_transform(ReduceCogniteModel(drop))
-
-    def include_referenced(self) -> IssueList:
-        """Include referenced views and containers in the data model."""
-        if self._state.client is None:
-            raise NeatSessionError(
-                "No client provided. You are referencing unknown views and containers in your data model, "
-                "NEAT needs a client to lookup the definitions. "
-                "Please set the client in the session, NeatSession(client=client)."
-            )
-        return self._state.rule_transform(IncludeReferenced(self._state.client))
+        return self._state.rule_transform(PrefixEntities(prefix))  # type: ignore[arg-type]

cognite/neat/_session/_read.py

@@ -4,10 +4,23 @@ from cognite.client.data_classes.data_modeling import DataModelId, DataModelIden
 from cognite.client.utils.useful_types import SequenceNotStr
 
 from cognite.neat._client import NeatClient
-from cognite.neat._constants import CLASSIC_CDF_NAMESPACE
+from cognite.neat._constants import (
+    CLASSIC_CDF_NAMESPACE,
+    get_default_prefixes_and_namespaces,
+)
 from cognite.neat._graph import examples as instances_examples
 from cognite.neat._graph import extractors
-from cognite.neat._graph.transformers import ConvertLiteral, LiteralToEntity, LookupRelationshipSourceTarget
+from cognite.neat._graph.transformers import (
+    ConvertLiteral,
+    LiteralToEntity,
+    Transformers,
+)
+from cognite.neat._graph.transformers._prune_graph import (
+    AttachPropertyFromTargetToSource,
+    PruneDeadEndEdges,
+    PruneInstancesOfUnknownType,
+    PruneTypes,
+)
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._issues.warnings import MissingCogniteClientWarning
@@ -99,20 +112,41 @@ class CDFReadAPI(BaseReadAPI):
         return self._state.rule_import(importer)
 
     def graph(
-        self, data_model_id: DataModelIdentifier, instance_space: str | SequenceNotStr[str] | None = None
+        self,
+        data_model_id: DataModelIdentifier,
+        instance_space: str | SequenceNotStr[str] | None = None,
+        skip_cognite_views: bool = True,
     ) -> IssueList:
         """Reads a knowledge graph from Cognite Data Fusion (CDF).
 
         Args:
             data_model_id: Tuple of strings with the id of a CDF Data Model.
             instance_space: The instance spaces to extract. If None, all instance spaces are extracted.
+            skip_cognite_views: If True, all Cognite Views are skipped. For example, if you have the CogniteAsset
+                view in you data model, it will ont be used to extract instances.
 
         Returns:
             IssueList: A list of issues that occurred during the extraction.
 
         """
+        return self._graph(data_model_id, instance_space, skip_cognite_views, unpack_json=False)
+
+    def _graph(
+        self,
+        data_model_id: DataModelIdentifier,
+        instance_space: str | SequenceNotStr[str] | None = None,
+        skip_cognite_views: bool = True,
+        unpack_json: bool = False,
+        str_to_ideal_type: bool = False,
+    ) -> IssueList:
         extractor = extractors.DMSGraphExtractor.from_data_model_id(
-            data_model_id, self._get_client, instance_space=instance_space
+            # We are skipping the Cognite Views
+            data_model_id,
+            self._get_client,
+            instance_space=instance_space,
+            skip_cognite_views=skip_cognite_views,
+            unpack_json=unpack_json,
+            str_to_ideal_type=str_to_ideal_type,
         )
         return self._state.write_graph(extractor)
 
@@ -130,7 +164,12 @@ class CDFClassicAPI(BaseReadAPI):
             raise ValueError("No client provided. Please provide a client to read a data model.")
         return self._state.client
 
-    def graph(self, root_asset_external_id: str, limit_per_type: int | None = None) -> IssueList:
+    def graph(
+        self,
+        root_asset_external_id: str,
+        limit_per_type: int | None = None,
+        identifier: Literal["id", "externalId"] = "id",
+    ) -> IssueList:
         """Reads the classic knowledge graph from CDF.
 
         The Classic Graph consists of the following core resource type.
@@ -165,6 +204,8 @@ class CDFClassicAPI(BaseReadAPI):
         Args:
             root_asset_external_id: The external id of the root asset
             limit_per_type: The maximum number of nodes to extract per core node type. If None, all nodes are extracted.
+            identifier: The identifier to use for the core nodes. Note selecting "id" can cause issues if the external
+                ID of the core nodes is missing. Default is "id".
 
         Returns:
             IssueList: A list of issues that occurred during the extraction.
@@ -174,6 +215,18 @@ class CDFClassicAPI(BaseReadAPI):
             neat.read.cdf.graph("root_asset_external_id")
             ```
         """
+        return self._graph(
+            root_asset_external_id, limit_per_type, identifier, reference_timeseries=False, reference_files=False
+        )
+
+    def _graph(
+        self,
+        root_asset_external_id: str,
+        limit_per_type: int | None = None,
+        identifier: Literal["id", "externalId"] = "id",
+        reference_timeseries: bool = False,
+        reference_files: bool = False,
+    ) -> IssueList:
         namespace = CLASSIC_CDF_NAMESPACE
         extractor = extractors.ClassicGraphExtractor(
             self._get_client,
@@ -181,14 +234,12 @@ class CDFClassicAPI(BaseReadAPI):
             limit_per_type=limit_per_type,
             namespace=namespace,
             prefix="Classic",
+            identifier=identifier,
         )
-        issues = self._state.write_graph(extractor)
-        issues.action = "Read Classic Graph"
-        if issues:
-            print("Use the .inspect.issues() for more details")
+        extract_issues = self._state.write_graph(extractor)
+        if identifier == "externalId":
+            self._state.quoted_source_identifiers = True
 
-        # Converting the instances from classic to core
-        self._state.instances.store.transform(LookupRelationshipSourceTarget(namespace, "Classic"))
         self._state.instances.store.transform(
             ConvertLiteral(
                 namespace["ClassicTimeSeries"],
@@ -200,11 +251,21 @@ class CDFClassicAPI(BaseReadAPI):
             LiteralToEntity(None, namespace["source"], "ClassicSourceSystem", "name"),
         )
         # Updating the information model.
-        self._state.rule_store.transform(ClassicPrepareCore(namespace))
+        prepare_issues = self._state.rule_store.transform(
+            ClassicPrepareCore(namespace, reference_timeseries, reference_files)
+        )
         # Update the instance store with the latest rules
         information_rules = self._state.rule_store.last_verified_information_rules
-        self._state.instances.store.rules = information_rules
-        return issues
+        self._state.instances.store.rules[self._state.instances.store.default_named_graph] = information_rules
+
+        all_issues = IssueList(extract_issues + prepare_issues)
+        # Update the provenance with all issue.
+        object.__setattr__(self._state.instances.store.provenance[-1].target_entity, "issues", all_issues)
+        all_issues.action = "Read Classic Graph"
+        if all_issues:
+            print("Use the .inspect.issues() for more details")
+
+        return all_issues
 
 
 @session_class_wrapper
@@ -241,7 +302,6 @@ class ExcelReadAPI(BaseReadAPI):
 class ExcelExampleAPI(BaseReadAPI):
     """Used as example for reading some data model into the NeatSession."""
 
-    @property
     def pump_example(self) -> IssueList:
         """Reads the Hello World pump example into the NeatSession."""
         importer: importers.ExcelImporter = importers.ExcelImporter(catalog.hello_world_pump)
@@ -340,7 +400,7 @@ class XMLReadAPI(BaseReadAPI):
             raise NeatValueError("Only support XML files of DEXPI format at the moment.")
 
     def dexpi(self, io: Any) -> None:
-        """Reads a DEXPI file into the NeatSession.
+        """Reads a DEXPI file into the NeatSession and executes set of predefined transformations.
 
         Args:
             io: file path or url to the DEXPI file
@@ -349,6 +409,13 @@ class XMLReadAPI(BaseReadAPI):
             ```python
             neat.read.xml.dexpi("url_or_path_to_dexpi_file")
             ```
+
+        !!! note "This method bundles several graph transformers which"
+            - attach values of generic attributes to nodes
+            - create associations between nodes
+            - remove unused generic attributes
+            - remove associations between nodes that do not exist in the extracted graph
+            - remove edges to nodes that do not exist in the extracted graph
         """
         path = NeatReader.create(io).materialize_path()
         engine = import_engine()
@@ -357,8 +424,36 @@ class XMLReadAPI(BaseReadAPI):
         extractor = engine.create_extractor()
         self._state.instances.store.write(extractor)
 
+        DEXPI = get_default_prefixes_and_namespaces()["dexpi"]
+
+        transformers = [
+            # Remove any instance which type is unknown
+            PruneInstancesOfUnknownType(),
+            # Directly connect generic attributes
+            AttachPropertyFromTargetToSource(
+                target_property=DEXPI.Value,
+                target_property_holding_new_property=DEXPI.Name,
+                target_node_type=DEXPI.GenericAttribute,
+                delete_target_node=True,
+            ),
+            # Directly connect associations
+            AttachPropertyFromTargetToSource(
+                target_property=DEXPI.ItemID,
+                target_property_holding_new_property=DEXPI.Type,
+                target_node_type=DEXPI.Association,
+                delete_target_node=True,
+            ),
+            # Remove unused generic attributes and associations
+            PruneTypes([DEXPI.GenericAttribute, DEXPI.Association]),
+            # Remove edges to nodes that do not exist in the extracted graph
+            PruneDeadEndEdges(),
+        ]
+
+        for transformer in transformers:
+            self._state.instances.store.transform(cast(Transformers, transformer))
+
     def aml(self, io: Any):
-        """Reads an AML file into NeatSession.
+        """Reads an AML file into NeatSession and executes a set of predefined transformations.
 
         Args:
             io: file path or url to the AML file
@@ -367,6 +462,11 @@ class XMLReadAPI(BaseReadAPI):
             ```python
             neat.read.xml.aml("url_or_path_to_aml_file")
             ```
+
+        !!! note "This method bundles several graph transformers which"
+            - attach values of attributes to nodes
+            - remove unused attributes
+            - remove edges to nodes that do not exist in the extracted graph
         """
         path = NeatReader.create(io).materialize_path()
         engine = import_engine()
@@ -375,6 +475,27 @@ class XMLReadAPI(BaseReadAPI):
         extractor = engine.create_extractor()
         self._state.instances.store.write(extractor)
 
+        AML = get_default_prefixes_and_namespaces()["aml"]
+
+        transformers = [
+            # Remove any instance which type is unknown
+            PruneInstancesOfUnknownType(),
+            # Directly connect generic attributes
+            AttachPropertyFromTargetToSource(
+                target_property=AML.Value,
+                target_property_holding_new_property=AML.Name,
+                target_node_type=AML.Attribute,
+                delete_target_node=True,
+            ),
+            # Prune unused attributes
+            PruneTypes([AML.Attribute]),
+            # # Remove edges to nodes that do not exist in the extracted graph
+            PruneDeadEndEdges(),
+        ]
+
+        for transformer in transformers:
+            self._state.instances.store.transform(cast(Transformers, transformer))
+
 
 @session_class_wrapper
 class RDFReadAPI(BaseReadAPI):

cognite/neat/_session/_set.py

@@ -3,9 +3,12 @@ from cognite.client import data_modeling as dm
 
 from cognite.neat._client import NeatClient
 from cognite.neat._constants import COGNITE_MODELS
+from cognite.neat._graph.transformers import SetType
 from cognite.neat._issues import IssueList
+from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._rules.models import DMSRules
 from cognite.neat._rules.transformers import SetIDDMSModel
+from cognite.neat._utils.text import humanize_collection
 
 from ._state import SessionState
 from .exceptions import NeatSessionError, session_class_wrapper
@@ -18,6 +21,7 @@ class SetAPI:
     def __init__(self, state: SessionState, verbose: bool) -> None:
         self._state = state
         self._verbose = verbose
+        self.instances = SetInstances(state, verbose)
 
     def data_model_id(self, new_model_id: dm.DataModelId | tuple[str, str, str]) -> IssueList:
         """Sets the data model ID of the latest verified data model. Set the data model id as a tuple of strings
@@ -29,7 +33,9 @@ class SetAPI:
             neat.set.data_model_id(("my_data_model_space", "My_Data_Model", "v1"))
             ```
         """
-        rules = self._state.rule_store.get_last_successful_entity().result
+        if self._state.rule_store.empty:
+            raise NeatSessionError("No rules to set the data model ID.")
+        rules = self._state.rule_store.provenance[-1].target_entity.dms
         if isinstance(rules, DMSRules):
             if rules.metadata.as_data_model_id() in COGNITE_MODELS:
                 raise NeatSessionError(
@@ -44,3 +50,46 @@ class SetAPI:
         if self._verbose:
             print(f"Client set to {self._state.client.config.project} CDF project.")
         return None
+
+
+@session_class_wrapper
+class SetInstances:
+    """Used to change instances"""
+
+    def __init__(self, state: SessionState, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+
+    def type_using_property(self, current_type: str, property_type: str, drop_property: bool = True) -> None:
+        """Replaces the type of all instances with the value of a property.
+
+        Example:
+            All Assets have a property `assetCategory` that we want to use as the type of all asset instances.
+
+            ```python
+            neat.set.instances.replace_type("Asset", "assetCategory")
+            ```
+        """
+        type_uri = self._state.instances.store.queries.type_uri(current_type)
+        property_uri = self._state.instances.store.queries.property_uri(property_type)
+
+        if not type_uri:
+            raise NeatValueError(f"Type {current_type} does not exist in the graph.")
+        elif len(type_uri) > 1:
+            raise NeatValueError(
+                f"{current_type} has multiple ids found in the graph: {humanize_collection(type_uri)}."
+            )
+
+        if not property_uri:
+            raise NeatValueError(f"Property {property_type} does not exist in the graph.")
+        elif len(type_uri) > 1:
+            raise NeatValueError(
+                f"{property_type} has multiple ids found in the graph: {humanize_collection(property_uri)}."
+            )
+
+        if not self._state.instances.store.queries.type_with_property(type_uri[0], property_uri[0]):
+            raise NeatValueError(f"Property {property_type} is not defined for type {current_type}.")
+
+        self._state.instances.store.transform(SetType(type_uri[0], property_uri[0], drop_property))
+
+        return None