cognite-neat 0.103.1__tar.gz → 0.104.0__tar.gz

{cognite_neat-0.103.1 → cognite_neat-0.104.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.103.1
+Version: 0.104.0
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

{cognite_neat-0.103.1 → cognite_neat-0.104.0}/cognite/neat/_graph/extractors/_mock_graph_generator.py

@@ -46,7 +46,7 @@ class MockGraphGenerator(BaseExtractor):
             # fixes potential issues with circular dependencies
             from cognite.neat._rules.transformers import DMSToInformation
 
-            self.rules = DMSToInformation().transform(rules).rules
+            self.rules = DMSToInformation().transform(rules)
         elif isinstance(rules, InformationRules):
             self.rules = rules
         else:

cognite_neat-0.104.0/cognite/neat/_graph/transformers/_base.py

@@ -0,0 +1,122 @@
+import dataclasses
+import warnings
+from abc import ABC, abstractmethod
+from typing import ClassVar, TypeAlias, cast
+
+from rdflib import Graph
+from rdflib.query import ResultRow
+
+from cognite.neat._issues.warnings import NeatValueWarning
+from cognite.neat._shared import Triple
+from cognite.neat._utils.collection_ import iterate_progress_bar
+from cognite.neat._utils.graph_transformations_report import GraphTransformationResult
+
+To_Add_Triples: TypeAlias = list[Triple]
+To_Remove_Triples: TypeAlias = list[Triple]
+
+
+@dataclasses.dataclass
+class RowTransformationOutput:
+    remove_triples: To_Remove_Triples = dataclasses.field(default_factory=list)
+    add_triples: To_Add_Triples = dataclasses.field(default_factory=list)
+    instances_removed_count: int = 0
+    instances_added_count: int = 0
+    instances_modified_count: int = 0
+
+
+class BaseTransformer(ABC):
+    description: str
+    _use_only_once: bool = False
+    _need_changes: ClassVar[frozenset[str]] = frozenset()
+
+    @abstractmethod
+    def transform(self, graph: Graph) -> None:
+        raise NotImplementedError()
+
+
+class BaseTransformerStandardised(ABC):
+    """Standardised base transformer to use in case a transformer is adding or removing triples from a graph. If you
+    are doing more specialised operations, please overwrite the .transform() method.
+    """
+
+    description: str
+    _use_only_once: bool = False
+    _need_changes: ClassVar[frozenset[str]] = frozenset()
+    _use_iterate_bar_threshold: int = 500
+
+    @abstractmethod
+    def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
+        """The operations to perform on each row resulting from the ._iterate_query() method.
+        The operation should return a list of triples to add and to remove.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def _count_query(self) -> str:
+        """
+        Overwrite to fetch all affected properties in the graph as a result of the transformation.
+        Returns:
+            A query string.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def _iterate_query(self) -> str:
+        """
+        The query to use for extracting target triples from the graph and performing the transformation.
+        Returns:
+            A query string.
+        """
+        raise NotImplementedError()
+
+    def _skip_count_query(self) -> str:
+        """
+        The query to use for extracting target triples from the graph and performing the transformation.
+        Returns:
+            A query string.
+        """
+        return ""
+
+    def transform(self, graph: Graph) -> GraphTransformationResult:
+        outcome = GraphTransformationResult(self.__class__.__name__)
+        outcome.added = outcome.modified = outcome.removed = 0
+
+        iteration_count_res = list(graph.query(self._count_query()))
+        iteration_count = int(iteration_count_res[0][0])  # type: ignore [index, arg-type]
+
+        outcome.affected_nodes_count = iteration_count
+
+        if self._skip_count_query():
+            skipped_count_res = list(graph.query(self._skip_count_query()))
+            skipped_count = int(skipped_count_res[0][0])  # type: ignore [index, arg-type]
+            warnings.warn(
+                NeatValueWarning(f"Skipping {skipped_count} properties in transformation {self.__class__.__name__}"),
+                stacklevel=2,
+            )
+            outcome.skipped = skipped_count
+
+        if iteration_count == 0:
+            return outcome
+
+        result_iterable = graph.query(self._iterate_query())
+        if iteration_count > self._use_iterate_bar_threshold:
+            result_iterable = iterate_progress_bar(  # type: ignore[misc, assignment]
+                result_iterable,
+                total=iteration_count,
+                description=self.description,
+            )
+
+        for row in result_iterable:
+            row = cast(ResultRow, row)
+            row_output = self.operation(row)
+
+            outcome.added += row_output.instances_added_count
+            outcome.removed += row_output.instances_removed_count
+            outcome.modified += row_output.instances_modified_count
+
+            for triple in row_output.add_triples:
+                graph.add(triple)
+            for triple in row_output.remove_triples:
+                graph.remove(triple)
+
+        return outcome

{cognite_neat-0.103.1 → cognite_neat-0.104.0}/cognite/neat/_graph/transformers/_classic_cdf.py

@@ -21,6 +21,7 @@ from cognite.neat._utils.rdf_ import (
 from ._base import BaseTransformer
 
 
+# TODO: standardise
 class AddAssetDepth(BaseTransformer):
     description: str = "Adds depth of asset in the asset hierarchy to the graph"
     _use_only_once: bool = True
@@ -84,6 +85,7 @@ class AddAssetDepth(BaseTransformer):
             return None
 
 
+# TODO: standardise
 class BaseAssetConnector(BaseTransformer, ABC):
     _asset_type: URIRef = DEFAULT_NAMESPACE.Asset
     _item_type: URIRef
@@ -166,6 +168,7 @@ class AssetEventConnector(BaseAssetConnector):
     _connection_type = DEFAULT_NAMESPACE.event
 
 
+# TODO: standardise
 class AssetRelationshipConnector(BaseTransformer):
     description: str = "Connects assets via relationships"
     _use_only_once: bool = True
@@ -242,6 +245,7 @@ class AssetRelationshipConnector(BaseTransformer):
                     graph.remove((relationship_id, self.relationship_target_xid_prop, None))
 
 
+# TODO: standardise
 class RelationshipAsEdgeTransformer(BaseTransformer):
     """Converts relationships into edges in the graph.
 

{cognite_neat-0.103.1 → cognite_neat-0.104.0}/cognite/neat/_graph/transformers/_prune_graph.py

@@ -1,15 +1,17 @@
 from typing import cast
 
 from rdflib import Graph, Namespace, URIRef
+from rdflib.query import ResultRow
 
 from cognite.neat._constants import DEFAULT_NAMESPACE
 from cognite.neat._shared import Triple
 from cognite.neat._utils.rdf_ import as_neat_compliant_uri
 from cognite.neat._utils.text import sentence_or_string_to_camel
 
-from ._base import BaseTransformer
+from ._base import BaseTransformer, BaseTransformerStandardised, RowTransformationOutput
 
 
+# TODO: Standardise after figuring out the bug which appears when running test_iodd_transformers.py
 class AttachPropertyFromTargetToSource(BaseTransformer):
     """
     Transformer that considers a TargetNode and SourceNode relationship, to extract a property that is attached to
@@ -148,6 +150,7 @@ class AttachPropertyFromTargetToSource(BaseTransformer):
                 graph.remove((target_node, None, None))
 
 
+# TODO: Remove or adapt IODD
 class PruneDanglingNodes(BaseTransformer):
     """
     Knowledge graph pruner and resolver. Will remove rdf triples from graph that does not have connections
@@ -189,18 +192,12 @@ class PruneDanglingNodes(BaseTransformer):
                 graph.remove((subject, None, None))
 
 
-class PruneTypes(BaseTransformer):
+class PruneTypes(BaseTransformerStandardised):
     """
     Removes all the instances of specific type
     """
 
     description: str = "Prunes nodes of specific rdf types"
-    _query_template = """
-                        SELECT ?subject
-                        WHERE {{
-                            ?subject a <{rdf_type}> .
-                            }}
-                      """
 
     def __init__(
         self,
@@ -208,50 +205,109 @@ class PruneTypes(BaseTransformer):
     ):
         self.node_prune_types = node_prune_types
 
-    def transform(self, graph: Graph) -> None:
-        for type_ in self.node_prune_types:
-            for (subject,) in list(graph.query(self._query_template.format(rdf_type=type_))):  # type: ignore
-                graph.remove((subject, None, None))
-
-
-class PruneDeadEndEdges(BaseTransformer):
+    def _iterate_query(self) -> str:
+        filter_string = ""
+        for node in self.node_prune_types:
+            filter_string += f" <{node}> "
+
+        query = """
+                SELECT ?subject
+                WHERE {{
+                  ?subject a ?type .
+                  VALUES ?type {{ {rdf_types_string} }}
+                }}
+                """
+        return query.format(rdf_types_string=filter_string)
+
+    def _count_query(self) -> str:
+        filter_string = ""
+        for node in self.node_prune_types:
+            filter_string += f" <{node}> "
+
+        query = """
+                SELECT ( COUNT( ?subject ) as ?count )
+                WHERE {{
+                  ?subject a ?type .
+                  VALUES ?type {{ {rdf_types_string} }}
+                }}
+                """
+        return query.format(rdf_types_string=filter_string)
+
+    def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
+        row_output = RowTransformationOutput()
+
+        (subject,) = query_result_row
+        row_output.remove_triples.append((subject, None, None))  # type: ignore
+        row_output.instances_removed_count = 1
+
+        return row_output
+
+
+class PruneDeadEndEdges(BaseTransformerStandardised):
     """
     Removes all the triples where object is a node that is not found in graph
     """
 
-    description: str = "Prunes the graph of specified rdf types that do not have connections to other nodes."
-    _query_template = """
-                        SELECT ?subject ?predicate ?object
-                        WHERE {
-                            ?subject ?predicate ?object .
-                            FILTER (isIRI(?object) && ?predicate != rdf:type)
-                            FILTER NOT EXISTS {?object ?p ?o .}
-
-                            }
-
-                      """
-
-    def transform(self, graph: Graph) -> None:
-        for triple in graph.query(self._query_template):
-            graph.remove(cast(Triple, triple))
-
-
-class PruneInstancesOfUnknownType(BaseTransformer):
+    description: str = "Pruning the graph of triples where object is a node that is not found in graph."
+
+    def _iterate_query(self) -> str:
+        return """
+                 SELECT ?subject ?predicate ?object
+                 WHERE {
+                     ?subject ?predicate ?object .
+                     FILTER (isIRI(?object) && ?predicate != rdf:type)
+                     FILTER NOT EXISTS {?object ?p ?o .}
+                     }
+                 """
+
+    def _count_query(self) -> str:
+        return """
+                SELECT (COUNT(?object) AS ?count)
+                WHERE {
+                    ?subject ?predicate ?object .
+                    FILTER (isIRI(?object) && ?predicate != rdf:type)
+                    FILTER NOT EXISTS {?object ?p ?o .}
+                    }
+                """
+
+    def operation(self, row: ResultRow) -> RowTransformationOutput:
+        row_output = RowTransformationOutput()
+        row_output.remove_triples.append(cast(Triple, row))
+        row_output.instances_modified_count = 1
+
+        return row_output
+
+
+class PruneInstancesOfUnknownType(BaseTransformerStandardised):
     """
     Removes all the triples where object is a node that is not found in graph
     """
 
-    description: str = "Prunes the graph of specified rdf types that do not have connections to other nodes."
-    _query_template = """
-                    SELECT DISTINCT ?subject
-                    WHERE {
-                        ?subject ?p ?o .
-                        FILTER NOT EXISTS {?subject a ?object .}
-
-                        }
-
-                    """
-
-    def transform(self, graph: Graph) -> None:
-        for (subject,) in graph.query(self._query_template):  # type: ignore
-            graph.remove((subject, None, None))
+    description: str = "Prunes the graph of triples where the object is a node that is not found in the graph."
+
+    def _iterate_query(self) -> str:
+        return """
+                SELECT DISTINCT ?subject
+                WHERE {
+                    ?subject ?p ?o .
+                    FILTER NOT EXISTS {?subject a ?object .}
+
+                    }
+                """
+
+    def _count_query(self) -> str:
+        return """
+                SELECT (COUNT(DISTINCT ?subject) as ?count)
+                WHERE {
+                    ?subject ?p ?o .
+                    FILTER NOT EXISTS {?subject a ?object .}
+                    }
+                """
+
+    def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
+        row_output = RowTransformationOutput()
+        (subject,) = query_result_row
+        row_output.remove_triples.append(cast(Triple, (subject, None, None)))
+        row_output.instances_removed_count = 1
+
+        return row_output

{cognite_neat-0.103.1 → cognite_neat-0.104.0}/cognite/neat/_graph/transformers/_rdfpath.py

@@ -2,15 +2,16 @@ from typing import cast
 from urllib.parse import quote
 
 from rdflib import Graph, URIRef
+from rdflib.query import ResultRow
 
 from cognite.neat._constants import DEFAULT_NAMESPACE
 from cognite.neat._rules.analysis import InformationAnalysis
 from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
 from cognite.neat._rules.models.information import InformationRules
 from cognite.neat._shared import Triple
-from cognite.neat._utils.rdf_ import add_triples_in_batch, remove_namespace_from_uri
+from cognite.neat._utils.rdf_ import remove_namespace_from_uri
 
-from ._base import BaseTransformer
+from ._base import BaseTransformer, BaseTransformerStandardised, RowTransformationOutput
 
 
 class ReduceHopTraversal(BaseTransformer):
@@ -19,6 +20,7 @@ class ReduceHopTraversal(BaseTransformer):
     ...
 
 
+# TODO: Standardise
 class AddSelfReferenceProperty(BaseTransformer):
     description: str = "Adds property that contains id of reference to all references of given class in Rules"
     _use_only_once: bool = True
@@ -56,17 +58,10 @@ class AddSelfReferenceProperty(BaseTransformer):
             property_.transformation = RDFPath(traversal=traversal)
 
 
-class MakeConnectionOnExactMatch(BaseTransformer):
+class MakeConnectionOnExactMatch(BaseTransformerStandardised):
     description: str = "Adds property that contains id of reference to all references of given class in Rules"
     _use_only_once: bool = True
     _need_changes = frozenset({})
-    _ref_template: str = """SELECT DISTINCT ?subject ?object
-                            WHERE {{
-                                ?subject a <{subject_type}> .
-                                ?subject <{subject_predicate}> ?value .
-                                ?object <{object_predicate}> ?value .
-                                ?object a <{object_type}> .
-                            }}"""
 
     def __init__(
         self,
@@ -90,20 +85,49 @@ class MakeConnectionOnExactMatch(BaseTransformer):
 
         self.limit = limit
 
-    def transform(self, graph: Graph) -> None:
-        query = self._ref_template.format(
+    def _iterate_query(self) -> str:
+        query = """SELECT DISTINCT ?subject ?object
+                            WHERE {{
+                                ?subject a <{subject_type}> .
+                                ?subject <{subject_predicate}> ?value .
+                                ?object <{object_predicate}> ?value .
+                                ?object a <{object_type}> .
+                            }}"""
+
+        if self.limit and isinstance(self.limit, int) and self.limit > 0:
+            query += f" LIMIT {self.limit}"
+
+        return query.format(
             subject_type=self.subject_type,
             subject_predicate=self.subject_predicate,
             object_type=self.object_type,
             object_predicate=self.object_predicate,
         )
 
+    def _count_query(self) -> str:
+        query = """SELECT (COUNT(DISTINCT (?subject ?object)) as ?count)
+                    WHERE {{
+                        ?subject a <{subject_type}> .
+                        ?subject <{subject_predicate}> ?value .
+                        ?object <{object_predicate}> ?value .
+                        ?object a <{object_type}> .
+                    }}"""
+
         if self.limit and isinstance(self.limit, int) and self.limit > 0:
             query += f" LIMIT {self.limit}"
 
-        triples: list[Triple] = []
-        for subject, object in graph.query(query):  # type: ignore [misc]
-            triples.append(cast(Triple, (subject, self.connection, object)))
+        return query.format(
+            subject_type=self.subject_type,
+            subject_predicate=self.subject_predicate,
+            object_type=self.object_type,
+            object_predicate=self.object_predicate,
+        )
+
+    def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
+        row_output = RowTransformationOutput()
+
+        subject, object = query_result_row
 
-        print(f"Found {len(triples)} connections. Adding them to the graph...")
-        add_triples_in_batch(graph, triples)
+        row_output.add_triples.append(cast(Triple, (subject, self.connection, object)))
+        row_output.instances_modified_count += 1
+        return row_output