cognite-neat 0.100.0__py3-none-any.whl → 0.101.0__py3-none-any.whl

cognite/neat/_client/_api/data_modeling_loaders.py

@@ -3,7 +3,7 @@ from abc import ABC, abstractmethod
 from collections.abc import Callable, Collection, Iterable, Sequence
 from dataclasses import dataclass, field
 from graphlib import TopologicalSorter
-from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar, cast
+from typing import TYPE_CHECKING, Any, ClassVar, Generic, Literal, TypeVar, cast
 
 from cognite.client.data_classes import filters
 from cognite.client.data_classes._base import (
@@ -363,9 +363,16 @@ class SpaceLoader(DataModelingLoader[str, SpaceApply, Space, SpaceApplyList, Spa
         return SpaceApplyList(schema.spaces.values())
 
     def has_data(self, item_id: str) -> bool:
-        return bool(self._client.data_modeling.instances.list("node", limit=1, space=item_id)) or bool(
-            self._client.data_modeling.instances.list("edge", limit=1, space=item_id)
-        )
+        if self._client.data_modeling.instances.list("node", limit=1, space=item_id):
+            return True
+        if self._client.data_modeling.instances.list("edge", limit=1, space=item_id):
+            return True
+        # Need to check if there are any containers with data in the space. Typically,
+        # a schema space will not contain data, while it will have containers that have data in an instance space.
+        for container in self._client.data_modeling.containers(space=item_id, include_global=False):
+            if self._client.loaders.containers.has_data(container.as_id()):
+                return True
+        return False
 
 
 class ContainerLoader(DataModelingLoader[ContainerId, ContainerApply, Container, ContainerApplyList, ContainerList]):
@@ -490,10 +497,23 @@ class ContainerLoader(DataModelingLoader[ContainerId, ContainerApply, Container,
         return ContainerApplyList(schema.containers.values())
 
     def has_data(self, item_id: ContainerId) -> bool:
-        has_data = filters.HasData(containers=[item_id])
-        return bool(self._client.data_modeling.instances.list("node", limit=1, filter=has_data)) or bool(
-            self._client.data_modeling.instances.list("edge", limit=1, filter=has_data)
-        )
+        has_data_filter = filters.HasData(containers=[item_id])
+        has_data = False
+        instance_type: Literal["node", "edge"]
+        # Mypy does not understand that the instance type is Literal["node", "edge"]
+        for instance_type in ["node", "edge"]:  # type: ignore[assignment]
+            try:
+                has_data = bool(
+                    self._client.data_modeling.instances.list(instance_type, limit=1, filter=has_data_filter)
+                )
+            except CogniteAPIError as e:
+                if e.code != 400:
+                    # If the container is used for nodes and we ask for edges, we get a 400 error. This
+                    # means there is no edge data for this container.
+                    raise
+            if has_data:
+                return True
+        return has_data
 
 
 class ViewLoader(DataModelingLoader[ViewId, ViewApply, View, ViewApplyList, ViewList]):

cognite/neat/_graph/loaders/_rdf2dms.py

@@ -156,7 +156,6 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
 
             for identifier, properties in reader:
                 try:
-                    print(view_id)
                     yield self._create_node(identifier, properties, pydantic_cls, view_id)
                 except ValueError as e:
                     error_node = ResourceCreationError(identifier, "node", error=str(e))

cognite/neat/_graph/transformers/__init__.py

@@ -7,6 +7,7 @@ from ._classic_cdf import (
     AssetTimeSeriesConnector,
     RelationshipAsEdgeTransformer,
 )
+from ._prune_graph import AttachPropertyFromTargetToSource, PruneDanglingNodes
 from ._rdfpath import AddSelfReferenceProperty, MakeConnectionOnExactMatch
 from ._value_type import SplitMultiValueProperty
 
@@ -21,6 +22,8 @@ __all__ = [
     "SplitMultiValueProperty",
     "RelationshipAsEdgeTransformer",
     "MakeConnectionOnExactMatch",
+    "AttachPropertyFromTargetToSource",
+    "PruneDanglingNodes",
 ]
 
 Transformers = (
@@ -34,4 +37,6 @@ Transformers = (
     | SplitMultiValueProperty
     | RelationshipAsEdgeTransformer
     | MakeConnectionOnExactMatch
+    | AttachPropertyFromTargetToSource
+    | PruneDanglingNodes
 )

cognite/neat/_graph/transformers/_iodd.py

@@ -2,12 +2,12 @@ from rdflib import Namespace
 
 from cognite.neat._graph.extractors import IODDExtractor
 
-from ._prune_graph import PruneDanglingNodes, TwoHopFlattener
+from ._prune_graph import AttachPropertyFromTargetToSource, PruneDanglingNodes
 
 IODD = Namespace("http://www.io-link.com/IODD/2010/10/")
 
 
-class IODDTwoHopFlattener(TwoHopFlattener):
+class IODDAttachPropertyFromTargetToSource(AttachPropertyFromTargetToSource):
     _need_changes = frozenset(
         {
             str(IODDExtractor.__name__),
@@ -15,11 +15,16 @@ class IODDTwoHopFlattener(TwoHopFlattener):
     )
 
     def __init__(self):
-        super().__init__(destination_node_type=IODD.TextObject, property_predicate=IODD.value, property_name="value")
+        super().__init__(
+            target_node_type=IODD.TextObject,
+            namespace=IODD,
+            target_property="value",
+            delete_target_node=True,
+        )
 
 
 class IODDPruneDanglingNodes(PruneDanglingNodes):
-    _need_changes = frozenset({str(IODDExtractor.__name__), str(IODDTwoHopFlattener.__name__)})
+    _need_changes = frozenset({str(IODDExtractor.__name__), str(IODDAttachPropertyFromTargetToSource.__name__)})
 
     def __init__(self):
         super().__init__(node_prune_types=[IODD.TextObject])

cognite/neat/_graph/transformers/_prune_graph.py

@@ -1,84 +1,141 @@
 from rdflib import Graph, Namespace, URIRef
 from rdflib.query import ResultRow
-from rdflib.term import Identifier
+
+from cognite.neat._utils.rdf_ import as_neat_compliant_uri
+from cognite.neat._utils.text import sentence_or_string_to_camel
 
 from ._base import BaseTransformer
 
 
-# TODO: Handle the cse when value is None, which will not make the TextObject resolve
-class TwoHopFlattener(BaseTransformer):
+class AttachPropertyFromTargetToSource(BaseTransformer):
     """
-    Transformer that will flatten the distance between a source node, an intermediate connecting node, and a
-    target property that is connected to the intermediate node.
-    The transformation result is that the target property is attached directly to the source node, instead of having
-    to go via the intermediate node.
-    The user can also provide a flag to decide if the intermediate node should be removed from the graph or not
-    after connecting the target property to the source node.
+    Transformer that considers a TargetNode and SourceNode relationship, to extract a property that is attached to
+    the TargetNode, and attaches it to the SourceNode instead, while also deleting the edge between
+    the SourceNode and TargetNode.
+    This means that you no longer have to go via the SourceNode to TargetNode to extract
+    the desired property from TargetNode, you can get it directly from the SourceNode instead.
+    Further, there are two ways of defining the predicate for the new property to attach to
+    the SourceNode. The predicate that is used will either be the old predicate between the SourceNode and TargetNode,
+    or, the TargetNode may hold a property with a value for the new predicate to use.
+    In this case, the user must specify the name of this predicate property connected to the TargetNode.
+    Consider the following example for illustration:
+
+        Ex. AttachPropertyFromTargetToSource
+        Graph before transformation:
+
+            :SourceNode a :SourceType .
+            :SourceNode :sourceProperty :TargetNode .
+
+            :TargetNode a :TargetType .
+            :TargetNode :propertyWhichValueWeWant 'Target Value' .
+            :TargetNode :propertyWhichValueWeMightWantAsNameForNewProperty 'PropertyName'
+
+        Use case A after transformation - attach new property to SourceNode using old predicate:
+
+            :SourceNode a :SourceType .
+            :SourceNode :sourceProperty 'Target Value' .
+
+        Use case B after transformation - extract new predicate from one of the properties of the TargetNode:
+
+            :SourceNode a :SourceType .
+            :SourceNode :PropertyName 'Target Value' .
 
-        Ex. TwoHopFlattener:
 
-        Graph before flattening (with deletion of intermediate node):
-        node(A, rdf:type(Pump)) -(predicate("vendor"))>
-                                node(B, rdf:type(TextObject)) -(predicate("value"))> Literal("CompanyX")
+    The user can provide a flag to decide if the intermediate target node should be removed from the graph or not
+    after connecting the target property to the source node. The example illustrates this.
+    The default however is False.
 
-        Graph after flattening nodes with destination_node_type = rdf:type(TextObject), property_predicate = :value,
-        and property_name = "value":
+    If delete_target_node is not set, the expected number of triples after this transformation should be the same as
+    before the transformation.
 
-        node(A, rdf:type(Pump)) -(predicate("vendor"))> Literal("CompanyX")
+    If delete_target_node is set, the expected number of triples should be:
+        #triples_before - #target_nodes * #target_nodes_properties
+
+        Number of triples after operation from above example: 5 - 1*3 = 2
 
     Args:
-        destination_node_type: RDF.type of edge Node
-        property_predicate: Predicate to use when resolving the value from the edge node
-        property_name: name of the property that the intermediate node is pointing to
-        delete_connecting_node: bool if the intermediate Node and Edge between source Node
-                                and target property should be deleted. Defaults to True.
+        target_node_type: RDF.type of edge Node
+        namespace: RDF Namespace to use when querying the graph
+        target_property: str with name of the property that holds the value attached to the intermediate node
+        target_property_holding_new_property_name: Optional str of the property name that holds
+        the new predicate to use when attaching the new property to the source node.
+        delete_target_node: bool if the intermediate Node and Edge between source Node
+                                and target property should be deleted. Defaults to False.
     """
 
-    description: str = "Prunes the graph of specified node types that do not have connections to other nodes."
-    _query_template: str = """SELECT ?sourceNode ?property ?destinationNode ?value WHERE {{
-                                     ?sourceNode ?property ?destinationNode .
-                                     ?destinationNode a <{destination_node_type}> .
-                                     ?destinationNode <{property_predicate}> ?{property_name} . }}"""
+    description: str = "Attaches a target property from a target node that is connected to a source node."
+
+    _query_template_use_case_a: str = """
+    SELECT ?sourceNode ?sourceProperty ?targetNode ?newSourceProperty ?newSourcePropertyValue WHERE {{
+        ?sourceNode ?sourceProperty ?targetNode .
+        BIND( <{target_property}> as ?newSourceProperty ) .
+        ?targetNode a <{target_node_type}> .
+        ?targetNode <{target_property}> ?newSourcePropertyValue . }}"""
+
+    _query_template_use_case_b: str = """
+    SELECT ?sourceNode ?sourceProperty ?targetNode ?newSourceProperty ?newSourcePropertyValue WHERE {{
+        ?sourceNode ?sourceProperty ?targetNode .
+        ?targetNode a <{target_node_type}> .
+        ?targetNode <{target_property_holding_new_property_name}> ?newSourceProperty .
+        ?targetNode <{target_property}> ?newSourcePropertyValue . }}"""
 
     def __init__(
         self,
-        destination_node_type: URIRef,
-        property_predicate: Namespace,
-        property_name: str,
-        delete_connecting_node: bool = True,
+        target_node_type: URIRef,
+        namespace: Namespace,
+        target_property: str,
+        target_property_holding_new_property_name: str | None = None,
+        delete_target_node: bool = False,
     ):
-        self.destination_node_type = destination_node_type
-        self.property_predicate = property_predicate
-        self.property_name = property_name
-        self.delete_connecting_node = delete_connecting_node
-
-    def transform(self, graph: Graph) -> None:
-        nodes_to_delete: list[Identifier] = []
-
-        graph_traversals = list(
-            graph.query(
-                self._query_template.format(
-                    destination_node_type=self.destination_node_type,
-                    property_predicate=self.property_predicate,
-                    property_name=self.property_name,
-                )
+        self.target_node_type = target_node_type
+        self.namespace = namespace
+        self.target_property = self.namespace[target_property]
+        self.delete_target_node = delete_target_node
+        self.target_property_holding_new_property_name = target_property_holding_new_property_name
+
+    def transform(self, graph) -> None:
+        nodes_to_delete: list[tuple] = []
+
+        if self.target_property_holding_new_property_name is not None:
+            target_property_holding_new_property_name = self.namespace[self.target_property_holding_new_property_name]
+            query = self._query_template_use_case_b.format(
+                target_node_type=self.target_node_type,
+                target_property_holding_new_property_name=target_property_holding_new_property_name,
+                target_property=self.target_property,
+            )
+        else:
+            query = self._query_template_use_case_a.format(
+                target_node_type=self.target_node_type,
+                target_property=self.target_property,
             )
-        )
-
-        for path in graph_traversals:
-            if isinstance(path, ResultRow):
-                source_node, predicate, destination_node, property_value = path.asdict().values()
-
-                # Create new connection from source node to value
-                graph.add((source_node, predicate, property_value))
-                nodes_to_delete.append(destination_node)
-
-            if self.delete_connecting_node:
-                for node in nodes_to_delete:
-                    # Remove edge triples to node
-                    graph.remove((None, None, node))
-                    # Remove node triple
-                    graph.remove((node, None, None))
+
+        for (
+            source_node,
+            old_predicate,
+            target_node,
+            new_predicate_value,
+            new_property_value,
+        ) in graph.query(query):
+            if self.target_property_holding_new_property_name is not None:
+                # Ensure new predicate is URI compliant as we are creating a new predicate
+                new_predicate_value_string = sentence_or_string_to_camel(str(new_predicate_value))
+                predicate = as_neat_compliant_uri(self.namespace[new_predicate_value_string])
+            else:
+                predicate = old_predicate
+
+            # Create new connection from source node to value
+            graph.add((source_node, predicate, new_property_value))
+            # Remove old relationship between source node and destination node
+            graph.remove((source_node, old_predicate, target_node))
+
+            nodes_to_delete.append(target_node)
+
+        if self.delete_target_node:
+            for target_node in nodes_to_delete:
+                # Remove triples with edges to target_node
+                graph.remove((None, None, target_node))
+                # Remove target node triple and its properties
+                graph.remove((target_node, None, None))
 
 
 class PruneDanglingNodes(BaseTransformer):

cognite/neat/_rules/exporters/_rules2dms.py

@@ -12,6 +12,7 @@ from cognite.client.data_classes._base import (
 from cognite.client.data_classes.data_modeling import (
     DataModelApplyList,
     DataModelId,
+    SpaceApply,
     ViewApplyList,
 )
 from cognite.client.exceptions import CogniteAPIError
@@ -200,6 +201,32 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
             loader.resource_name for loader, categorized in categorized_items_by_loader.items() if categorized.to_update
         )
 
+        deleted_by_name: dict[str, UploadResult] = {}
+        if not is_failing:
+            # Deletion is done in reverse order to take care of dependencies
+            for loader, items in reversed(categorized_items_by_loader.items()):
+                issue_list = IssueList()
+
+                if items.resource_name == client.loaders.data_models.resource_name:
+                    warning_list = self._validate(list(items.item_ids()), client)
+                    issue_list.extend(warning_list)
+
+                results = UploadResult(loader.resource_name, issues=issue_list)  # type: ignore[var-annotated]
+                if dry_run:
+                    results.deleted.update(items.to_delete_ids)
+                else:
+                    if items.to_delete_ids:
+                        try:
+                            deleted = loader.delete(items.to_delete_ids)
+                        except MultiCogniteAPIError as e:
+                            results.deleted.update([loader.get_id(item) for item in e.success])
+                            results.failed_deleted.update([loader.get_id(item) for item in e.failed])
+                            for error in e.errors:
+                                results.error_messages.append(f"Failed to delete {loader.resource_name}: {error!s}")
+                        else:
+                            results.deleted.update(deleted)
+                deleted_by_name[loader.resource_name] = results
+
         for loader, items in categorized_items_by_loader.items():
             issue_list = IssueList()
 
@@ -221,28 +248,21 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
 
             results.unchanged.update(items.unchanged_ids)
             results.skipped.update(items.to_skip_ids)
+            if delete_results := deleted_by_name.get(loader.resource_name):
+                results.deleted.update(delete_results.deleted)
+                results.failed_deleted.update(delete_results.failed_deleted)
+                results.error_messages.extend(delete_results.error_messages)
+
             if dry_run:
                 if self.existing in ["update", "force"]:
                     # Assume all changed are successful
                     results.changed.update(items.to_update_ids)
                 elif self.existing == "skip":
                     results.skipped.update(items.to_update_ids)
-                results.deleted.update(items.to_delete_ids)
                 results.created.update(items.to_create_ids)
                 yield results
                 continue
 
-            if items.to_delete_ids:
-                try:
-                    deleted = loader.delete(items.to_delete_ids)
-                except MultiCogniteAPIError as e:
-                    results.deleted.update([loader.get_id(item) for item in e.success])
-                    results.failed_deleted.update([loader.get_id(item) for item in e.failed])
-                    for error in e.errors:
-                        results.error_messages.append(f"Failed to delete {loader.resource_name}: {error!s}")
-                else:
-                    results.deleted.update(deleted)
-
             if items.to_create:
                 try:
                     created = loader.create(items.to_create)
@@ -304,12 +324,14 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
                 and not loader.in_space(item, self.include_space)
             ):
                 continue
-
-            cdf_item = cdf_item_by_id.get(loader.get_id(item))
+            item_id = loader.get_id(item)
+            cdf_item = cdf_item_by_id.get(item_id)
             if cdf_item is None:
                 categorized.to_create.append(item)
-            elif is_redeploying or self.existing == "recreate":
-                if loader.has_data(cdf_item) and not self.drop_data:
+            elif (is_redeploying or self.existing == "recreate") and not isinstance(item, SpaceApply):
+                # Spaces are not deleted, instead they are updated. Deleting a space is an expensive operation
+                # and are seldom needed. If you need to delete the space, it should be done in a different operation.
+                if not self.drop_data and loader.has_data(item_id):
                     categorized.to_skip.append(cdf_item)
                 else:
                     categorized.to_delete.append(cdf_item.as_write())

cognite/neat/_rules/models/_base_rules.py

@@ -147,11 +147,13 @@ class BaseMetadata(SchemaModel):
     Metadata model for data model
     """
 
-    role: ClassVar[RoleTypes]
-    aspect: ClassVar[DataModelAspect]
-    space: SpaceType = Field(alias="prefix")
-    external_id: DataModelExternalIdType = Field(alias="externalId")
-    version: VersionType
+    role: ClassVar[RoleTypes] = Field(description="Role of the person creating the data model")
+    aspect: ClassVar[DataModelAspect] = Field(description="Aspect of the data model")
+    space: SpaceType = Field(alias="prefix", description="The space where the data model is defined")
+    external_id: DataModelExternalIdType = Field(
+        alias="externalId", description="External identifier for the data model"
+    )
+    version: VersionType = Field(description="Version of the data model")
 
     name: str | None = Field(
         None,
@@ -160,21 +162,23 @@ class BaseMetadata(SchemaModel):
         max_length=255,
     )
 
-    description: str | None = Field(None, min_length=1, max_length=1024)
+    description: str | None = Field(
+        None, min_length=1, max_length=1024, description="Short description of the data model"
+    )
 
     creator: StrListType = Field(
         description=(
-            "List of contributors to the data model creation, "
+            "List of contributors (comma seperated) to the data model creation, "
             "typically information architects are considered as contributors."
         ),
     )
 
     created: datetime = Field(
-        description=("Date of the data model creation"),
+        description="Date of the data model creation",
     )
 
     updated: datetime = Field(
-        description=("Date of the data model update"),
+        description="Date of the data model update",
     )
 
     @field_validator("*", mode="before")