cognite-neat 0.101.0__py3-none-any.whl → 0.103.0__py3-none-any.whl

cognite/neat/_graph/transformers/_prune_graph.py

@@ -1,6 +1,9 @@
+from typing import cast
+
 from rdflib import Graph, Namespace, URIRef
-from rdflib.query import ResultRow
 
+from cognite.neat._constants import DEFAULT_NAMESPACE
+from cognite.neat._shared import Triple
 from cognite.neat._utils.rdf_ import as_neat_compliant_uri
 from cognite.neat._utils.text import sentence_or_string_to_camel
 
@@ -55,12 +58,13 @@ class AttachPropertyFromTargetToSource(BaseTransformer):
 
     Args:
         target_node_type: RDF.type of edge Node
-        namespace: RDF Namespace to use when querying the graph
-        target_property: str with name of the property that holds the value attached to the intermediate node
-        target_property_holding_new_property_name: Optional str of the property name that holds
-        the new predicate to use when attaching the new property to the source node.
+        target_property: URIRef of the property that holds the value attached to the intermediate node
+        target_property_holding_new_property: URIRef of the property which value will be new
+        property that will be added to the source node
         delete_target_node: bool if the intermediate Node and Edge between source Node
                                 and target property should be deleted. Defaults to False.
+        convert_literal_to_uri: bool if the value of the new property should be converted to URIRef. Defaults to False.
+        namespace: Namespace to use when converting value to URIRef. Defaults to DEFAULT_NAMESPACE.
     """
 
     description: str = "Attaches a target property from a target node that is connected to a source node."
@@ -82,25 +86,26 @@ class AttachPropertyFromTargetToSource(BaseTransformer):
     def __init__(
         self,
         target_node_type: URIRef,
-        namespace: Namespace,
-        target_property: str,
-        target_property_holding_new_property_name: str | None = None,
+        target_property: URIRef,
+        target_property_holding_new_property: URIRef | None = None,
         delete_target_node: bool = False,
+        convert_literal_to_uri: bool = False,
+        namespace: Namespace | None = None,
     ):
         self.target_node_type = target_node_type
-        self.namespace = namespace
-        self.target_property = self.namespace[target_property]
+        self.target_property = target_property
         self.delete_target_node = delete_target_node
-        self.target_property_holding_new_property_name = target_property_holding_new_property_name
+        self.target_property_holding_new_property = target_property_holding_new_property
+        self.convert_literal_to_uri = convert_literal_to_uri
+        self.namespace = namespace or DEFAULT_NAMESPACE
 
     def transform(self, graph) -> None:
         nodes_to_delete: list[tuple] = []
 
-        if self.target_property_holding_new_property_name is not None:
-            target_property_holding_new_property_name = self.namespace[self.target_property_holding_new_property_name]
+        if self.target_property_holding_new_property is not None:
             query = self._query_template_use_case_b.format(
                 target_node_type=self.target_node_type,
-                target_property_holding_new_property_name=target_property_holding_new_property_name,
+                target_property_holding_new_property_name=self.target_property_holding_new_property,
                 target_property=self.target_property,
             )
         else:
@@ -116,15 +121,20 @@ class AttachPropertyFromTargetToSource(BaseTransformer):
             new_predicate_value,
             new_property_value,
         ) in graph.query(query):
-            if self.target_property_holding_new_property_name is not None:
+            if self.target_property_holding_new_property is not None:
                 # Ensure new predicate is URI compliant as we are creating a new predicate
                 new_predicate_value_string = sentence_or_string_to_camel(str(new_predicate_value))
                 predicate = as_neat_compliant_uri(self.namespace[new_predicate_value_string])
             else:
                 predicate = old_predicate
-
             # Create new connection from source node to value
-            graph.add((source_node, predicate, new_property_value))
+            graph.add(
+                (
+                    source_node,
+                    predicate,
+                    (self.namespace[new_property_value] if self.convert_literal_to_uri else new_property_value),
+                )
+            )
             # Remove old relationship between source node and destination node
             graph.remove((source_node, old_predicate, target_node))
 
@@ -158,7 +168,7 @@ class PruneDanglingNodes(BaseTransformer):
         node_prune_types: list of RDF types to prune from the Graph if they are stand-alone Nodes
     """
 
-    description: str = "Prunes the graph of specified rdf types that do not have connections to other nodes."
+    description: str = "Prunes nodes of specific rdf types that do not have any connection to them."
     _query_template = """
                     SELECT ?subject
                     WHERE {{
@@ -174,10 +184,74 @@ class PruneDanglingNodes(BaseTransformer):
         self.node_prune_types = node_prune_types
 
     def transform(self, graph: Graph) -> None:
-        for object_type in self.node_prune_types:
-            nodes_without_neighbours = list(graph.query(self._query_template.format(rdf_type=object_type)))
+        for type_ in self.node_prune_types:
+            for (subject,) in list(graph.query(self._query_template.format(rdf_type=type_))):  # type: ignore
+                graph.remove((subject, None, None))
+
+
+class PruneTypes(BaseTransformer):
+    """
+    Removes all the instances of specific type
+    """
+
+    description: str = "Prunes nodes of specific rdf types"
+    _query_template = """
+                        SELECT ?subject
+                        WHERE {{
+                            ?subject a <{rdf_type}> .
+                            }}
+                      """
+
+    def __init__(
+        self,
+        node_prune_types: list[URIRef],
+    ):
+        self.node_prune_types = node_prune_types
+
+    def transform(self, graph: Graph) -> None:
+        for type_ in self.node_prune_types:
+            for (subject,) in list(graph.query(self._query_template.format(rdf_type=type_))):  # type: ignore
+                graph.remove((subject, None, None))
 
-            for node in nodes_without_neighbours:
-                # Remove node and its property triples in the graph
-                if isinstance(node, ResultRow):
-                    graph.remove((node["subject"], None, None))
+
+class PruneDeadEndEdges(BaseTransformer):
+    """
+    Removes all the triples where object is a node that is not found in graph
+    """
+
+    description: str = "Prunes the graph of specified rdf types that do not have connections to other nodes."
+    _query_template = """
+                        SELECT ?subject ?predicate ?object
+                        WHERE {
+                            ?subject ?predicate ?object .
+                            FILTER (isIRI(?object) && ?predicate != rdf:type)
+                            FILTER NOT EXISTS {?object ?p ?o .}
+
+                            }
+
+                      """
+
+    def transform(self, graph: Graph) -> None:
+        for triple in graph.query(self._query_template):
+            graph.remove(cast(Triple, triple))
+
+
+class PruneInstancesOfUnknownType(BaseTransformer):
+    """
+    Removes all the triples where object is a node that is not found in graph
+    """
+
+    description: str = "Prunes the graph of specified rdf types that do not have connections to other nodes."
+    _query_template = """
+                    SELECT DISTINCT ?subject
+                    WHERE {
+                        ?subject ?p ?o .
+                        FILTER NOT EXISTS {?subject a ?object .}
+
+                        }
+
+                    """
+
+    def transform(self, graph: Graph) -> None:
+        for (subject,) in graph.query(self._query_template):  # type: ignore
+            graph.remove((subject, None, None))

cognite/neat/_graph/transformers/_value_type.py

@@ -1,8 +1,17 @@
-from rdflib import XSD, Graph, URIRef
+import warnings
+from collections.abc import Callable
+from typing import Any, cast
+from urllib.parse import quote
+
+import rdflib
+from rdflib import RDF, XSD, Graph, Namespace, URIRef
 
 from cognite.neat._constants import UNKNOWN_TYPE
 from cognite.neat._graph.queries import Queries
-from cognite.neat._utils.rdf_ import remove_namespace_from_uri
+from cognite.neat._issues.warnings import NeatValueWarning, PropertyDataTypeConversionWarning
+from cognite.neat._utils.auxiliary import string_to_ideal_type
+from cognite.neat._utils.collection_ import iterate_progress_bar
+from cognite.neat._utils.rdf_ import get_namespace, remove_namespace_from_uri
 
 from ._base import BaseTransformer
 
@@ -64,3 +73,188 @@ class SplitMultiValueProperty(BaseTransformer):
                     graph.remove((s, property_uri, o))
                     new_property = URIRef(f"{property_uri}_{remove_namespace_from_uri(value_type_uri)}")
                     graph.add((s, new_property, o))
+
+
+class ConvertLiteral(BaseTransformer):
+    description: str = "ConvertLiteral is a transformer that improve data typing of a literal value."
+    _use_only_once: bool = False
+    _need_changes = frozenset({})
+
+    _count_by_properties = """SELECT (COUNT(?value) AS ?valueCount)
+    WHERE {{
+      ?instance a <{subject_type}> .
+      ?instance <{subject_predicate}> ?value
+       FILTER(isLiteral(?value))
+    }}"""
+
+    _count_by_properties_uri = """SELECT (COUNT(?value) AS ?valueCount)
+        WHERE {{
+          ?instance a <{subject_type}> .
+          ?instance <{subject_predicate}> ?value
+           FILTER(isIRI(?value))
+        }}"""
+
+    _properties = """SELECT ?instance ?value
+    WHERE {{
+      ?instance a <{subject_type}> .
+      ?instance <{subject_predicate}> ?value
+
+      FILTER(isLiteral(?value))
+
+    }}"""
+
+    def __init__(
+        self,
+        subject_type: URIRef,
+        subject_predicate: URIRef,
+        conversion: Callable[[Any], Any] | None = None,
+    ) -> None:
+        self.subject_type = subject_type
+        self.subject_predicate = subject_predicate
+        self.conversion = conversion or string_to_ideal_type
+        self._type_name = remove_namespace_from_uri(subject_type)
+        self._property_name = remove_namespace_from_uri(subject_predicate)
+
+    def transform(self, graph: Graph) -> None:
+        count_connection_query = self._count_by_properties_uri.format(
+            subject_type=self.subject_type, subject_predicate=self.subject_predicate
+        )
+        connection_count_res = list(graph.query(count_connection_query))
+        connection_count = int(connection_count_res[0][0])  # type: ignore [index, arg-type]
+
+        if connection_count > 0:
+            warnings.warn(
+                NeatValueWarning(
+                    f"Skipping {connection_count} of {self._type_name}.{self._property_name} "
+                    f"as these are connections and not data values."
+                ),
+                stacklevel=2,
+            )
+
+        count_query = self._count_by_properties.format(
+            subject_type=self.subject_type, subject_predicate=self.subject_predicate
+        )
+
+        property_count_res = list(graph.query(count_query))
+        property_count = int(property_count_res[0][0])  # type: ignore [index, arg-type]
+        iterate_query = self._properties.format(
+            subject_type=self.subject_type, subject_predicate=self.subject_predicate
+        )
+
+        for instance, literal in iterate_progress_bar(  # type: ignore[misc]
+            graph.query(iterate_query),
+            total=property_count,
+            description=f"Converting {self._type_name}.{self._property_name}.",
+        ):
+            value = cast(rdflib.Literal, literal).toPython()
+            try:
+                converted_value = self.conversion(value)
+            except Exception as e:
+                warnings.warn(
+                    PropertyDataTypeConversionWarning(str(instance), self._type_name, self._property_name, str(e)),
+                    stacklevel=2,
+                )
+                continue
+
+            graph.add((instance, self.subject_predicate, rdflib.Literal(converted_value)))
+            graph.remove((instance, self.subject_predicate, literal))
+
+
+class LiteralToEntity(BaseTransformer):
+    description = "Converts a literal value to new entity"
+
+    _count_properties_of_type = """SELECT (COUNT(?property) AS ?propertyCount)
+    WHERE {{
+      ?instance a <{subject_type}> .
+      ?instance <{subject_predicate}> ?property
+      FILTER(isLiteral(?property))
+    }}"""
+    _count_connections_of_type = """SELECT (COUNT(?property) AS ?propertyCount)
+    WHERE {{
+      ?instance a <{subject_type}> .
+      ?instance <{subject_predicate}> ?property
+      FILTER(isIRI(?property))
+    }}"""
+
+    _properties_of_type = """SELECT ?instance ?property
+    WHERE {{
+      ?instance a <{subject_type}> .
+      ?instance <{subject_predicate}> ?property
+      FILTER(isLiteral(?property))
+    }}"""
+
+    _count_properties = """SELECT (COUNT(?property) AS ?propertyCount)
+    WHERE {{
+      ?instance <{subject_predicate}> ?property
+      FILTER(isLiteral(?property))
+    }}"""
+    _count_connections = """SELECT (COUNT(?property) AS ?propertyCount)
+        WHERE {{
+          ?instance <{subject_predicate}> ?property
+          FILTER(isIRI(?property))
+        }}"""
+    _properties = """SELECT ?instance ?property
+    WHERE {{
+      ?instance <{subject_predicate}> ?property
+      FILTER(isLiteral(?property))
+    }}"""
+
+    def __init__(
+        self, subject_type: URIRef | None, subject_predicate: URIRef, entity_type: str, new_property: str | None = None
+    ) -> None:
+        self.subject_type = subject_type
+        self.subject_predicate = subject_predicate
+        self.entity_type = entity_type
+        self.new_property = new_property
+
+    def transform(self, graph: Graph) -> None:
+        if self.subject_type is None:
+            count_query = self._count_properties.format(subject_predicate=self.subject_predicate)
+            iterate_query = self._properties.format(subject_predicate=self.subject_predicate)
+            connection_count_query = self._count_connections.format(subject_predicate=self.subject_predicate)
+        else:
+            count_query = self._count_properties_of_type.format(
+                subject_type=self.subject_type, subject_predicate=self.subject_predicate
+            )
+            iterate_query = self._properties_of_type.format(
+                subject_type=self.subject_type, subject_predicate=self.subject_predicate
+            )
+            connection_count_query = self._count_connections_of_type.format(
+                subject_type=self.subject_type, subject_predicate=self.subject_predicate
+            )
+
+        connection_count_res = list(graph.query(connection_count_query))
+        connection_count = int(connection_count_res[0][0])  # type: ignore [index, arg-type]
+        if connection_count > 0:
+            warnings.warn(
+                NeatValueWarning(
+                    f"Skipping {connection_count} of {remove_namespace_from_uri(self.subject_predicate)} "
+                    f"as these are connections and not data values."
+                ),
+                stacklevel=2,
+            )
+
+        property_count_res = list(graph.query(count_query))
+        property_count = int(property_count_res[0][0])  # type: ignore [index, arg-type]
+
+        instance: URIRef
+        description = f"Creating {remove_namespace_from_uri(self.subject_predicate)}."
+        if self.subject_type is not None:
+            description = (
+                f"Creating {remove_namespace_from_uri(self.subject_type)}."
+                f"{remove_namespace_from_uri(self.subject_predicate)}."
+            )
+        for instance, literal in iterate_progress_bar(  # type: ignore[misc, assignment]
+            graph.query(iterate_query),
+            total=property_count,
+            description=description,
+        ):
+            value = cast(rdflib.Literal, literal).toPython()
+            namespace = Namespace(get_namespace(instance))
+            entity_type = namespace[self.entity_type]
+            new_entity = namespace[f"{self.entity_type}_{quote(value)!s}"]
+            graph.add((new_entity, RDF.type, entity_type))
+            if self.new_property is not None:
+                graph.add((new_entity, namespace[self.new_property], rdflib.Literal(value)))
+            graph.add((instance, self.subject_predicate, new_entity))
+            graph.remove((instance, self.subject_predicate, literal))

cognite/neat/_issues/__init__.py

@@ -15,14 +15,14 @@ from ._base import (
 )
 
 __all__ = [
-    "NeatIssue",
-    "NeatError",
-    "NeatWarning",
     "DefaultWarning",
-    "NeatIssueList",
+    "FutureResult",
     "IssueList",
     "MultiValueError",
-    "catch_warnings",
+    "NeatError",
+    "NeatIssue",
+    "NeatIssueList",
+    "NeatWarning",
     "catch_issues",
-    "FutureResult",
+    "catch_warnings",
 ]

cognite/neat/_issues/_base.py

@@ -32,12 +32,12 @@ else:
 
 
 __all__ = [
-    "NeatIssue",
-    "NeatError",
-    "NeatWarning",
     "DefaultWarning",
-    "NeatIssueList",
     "MultiValueError",
+    "NeatError",
+    "NeatIssue",
+    "NeatIssueList",
+    "NeatWarning",
 ]
 
 T_Identifier = TypeVar("T_Identifier", bound=Hashable)

cognite/neat/_issues/errors/__init__.py

@@ -38,40 +38,40 @@ from ._workflow import (
 )
 
 __all__ = [
+    "AuthorizationError",
+    "CDFMissingClientError",
+    "DefaultPydanticError",
+    "FileMissingRequiredFieldError",
+    "FileNotAFileError",
+    "FileNotFoundNeatError",
+    "FileReadError",
+    "FileTypeUnexpectedError",
     "NeatError",
-    "NeatValueError",
     "NeatImportError",
-    "RegexViolationError",
-    "AuthorizationError",
+    "NeatTypeError",
+    "NeatValueError",
     "NeatYamlError",
-    "FileReadError",
-    "ResourceCreationError",
-    "FileNotFoundNeatError",
-    "FileMissingRequiredFieldError",
+    "PropertyDefinitionDuplicatedError",
     "PropertyDefinitionError",
-    "PropertyTypeNotSupportedError",
+    "PropertyMappingDuplicatedError",
     "PropertyNotFoundError",
-    "PropertyDefinitionDuplicatedError",
+    "PropertyTypeNotSupportedError",
+    "RegexViolationError",
     "ResourceChangedError",
+    "ResourceConversionError",
+    "ResourceCreationError",
     "ResourceDuplicatedError",
-    "ResourceRetrievalError",
-    "ResourceNotFoundError",
     "ResourceError",
-    "ResourceNotDefinedError",
     "ResourceMissingIdentifierError",
-    "ResourceConversionError",
-    "WorkflowConfigurationNotSetError",
+    "ResourceNotDefinedError",
+    "ResourceNotFoundError",
+    "ResourceRetrievalError",
+    "ReversedConnectionNotFeasibleError",
+    "RowError",
     "WorkFlowMissingDataError",
+    "WorkflowConfigurationNotSetError",
     "WorkflowStepNotInitializedError",
     "WorkflowStepOutputError",
-    "FileTypeUnexpectedError",
-    "FileNotAFileError",
-    "DefaultPydanticError",
-    "PropertyMappingDuplicatedError",
-    "RowError",
-    "NeatTypeError",
-    "ReversedConnectionNotFeasibleError",
-    "CDFMissingClientError",
 ]
 
 _NEAT_ERRORS_BY_NAME = {error.__name__: error for error in _get_subclasses(NeatError, include_base=True)}

cognite/neat/_issues/formatters.py

@@ -5,7 +5,7 @@ from pathlib import Path
 
 from ._base import NeatError, NeatIssueList, NeatWarning
 
-__all__ = ["Formatter", "BasicHTML", "FORMATTER_BY_NAME"]
+__all__ = ["FORMATTER_BY_NAME", "BasicHTML", "Formatter"]
 
 
 class Formatter(ABC):

cognite/neat/_issues/warnings/__init__.py

@@ -26,6 +26,7 @@ from ._models import (
     UserModelingWarning,
 )
 from ._properties import (
+    PropertyDataTypeConversionWarning,
     PropertyDefinitionDuplicatedWarning,
     PropertyNotFoundWarning,
     PropertyOverwritingWarning,
@@ -43,38 +44,39 @@ from ._resources import (
 )
 
 __all__ = [
+    "BreakingModelingPrincipleWarning",
+    "CDFAuthWarning",
+    "CDFMaxIterationsWarning",
+    "CDFNotSupportedWarning",
     "DefaultWarning",
-    "FileReadWarning",
-    "FileMissingRequiredFieldWarning",
     "FileItemNotSupportedWarning",
+    "FileMissingRequiredFieldWarning",
+    "FileReadWarning",
     "FileTypeUnexpectedWarning",
     "NeatValueWarning",
+    "NotSupportedHasDataFilterLimitWarning",
+    "NotSupportedViewContainerLimitWarning",
     "NotSupportedWarning",
-    "UserModelingWarning",
-    "CDFNotSupportedWarning",
-    "BreakingModelingPrincipleWarning",
+    "PrincipleMatchingSpaceAndVersionWarning",
+    "PrincipleOneModelOneSpaceWarning",
+    "PrincipleSolutionBuildsOnEnterpriseWarning",
+    "PropertyDataTypeConversionWarning",
     "PropertyDefinitionDuplicatedWarning",
-    "PropertyTypeNotSupportedWarning",
     "PropertyNotFoundWarning",
-    "PropertyValueTypeUndefinedWarning",
     "PropertyOverwritingWarning",
     "PropertySkippedWarning",
-    "ResourceNeatWarning",
-    "ResourcesDuplicatedWarning",
+    "PropertyTypeNotSupportedWarning",
+    "PropertyValueTypeUndefinedWarning",
     "RegexViolationWarning",
+    "ResourceNeatWarning",
     "ResourceNotFoundWarning",
-    "ResourceTypeNotSupportedWarning",
-    "ResourceRetrievalWarning",
     "ResourceRegexViolationWarning",
-    "PrincipleOneModelOneSpaceWarning",
-    "PrincipleMatchingSpaceAndVersionWarning",
-    "PrincipleSolutionBuildsOnEnterpriseWarning",
-    "NotSupportedViewContainerLimitWarning",
-    "NotSupportedHasDataFilterLimitWarning",
+    "ResourceRetrievalWarning",
+    "ResourceTypeNotSupportedWarning",
+    "ResourcesDuplicatedWarning",
     "UndefinedViewWarning",
-    "CDFAuthWarning",
+    "UserModelingWarning",
     "user_modeling",
-    "CDFMaxIterationsWarning",
 ]
 
 _NEAT_WARNINGS_BY_NAME = {warning.__name__: warning for warning in _get_subclasses(NeatWarning, include_base=True)}

cognite/neat/_issues/warnings/_properties.py

@@ -70,3 +70,10 @@ class PropertySkippedWarning(PropertyWarning[T_Identifier]):
     which is skipped. {reason}."""
 
     reason: str
+
+
+@dataclass(unsafe_hash=True)
+class PropertyDataTypeConversionWarning(PropertyWarning[T_Identifier]):
+    """The {resource_type} with identifier {identifier} failed to convert the property {property_name}: {error}"""
+
+    error: str

cognite/neat/_issues/warnings/user_modeling.py

@@ -12,12 +12,12 @@ from cognite.neat._constants import DMS_CONTAINER_PROPERTY_SIZE_LIMIT
 from ._models import UserModelingWarning
 
 __all__ = [
+    "ContainerPropertyLimitWarning",
     "DirectRelationMissingSourceWarning",
     "EmptyContainerWarning",
     "HasDataFilterOnNoPropertiesViewWarning",
-    "NodeTypeFilterOnParentViewWarning",
     "HasDataFilterOnViewWithReferencesWarning",
-    "ContainerPropertyLimitWarning",
+    "NodeTypeFilterOnParentViewWarning",
     "NotNeatSupportedFilterWarning",
     "ParentInDifferentSpaceWarning",
 ]

cognite/neat/_rules/analysis/__init__.py

@@ -1,4 +1,4 @@
 from ._dms import DMSAnalysis
 from ._information import InformationAnalysis
 
-__all__ = ["InformationAnalysis", "DMSAnalysis"]
+__all__ = ["DMSAnalysis", "InformationAnalysis"]

cognite/neat/_rules/catalog/__init__.py

@@ -4,3 +4,4 @@ from pathlib import Path
 
 _CATALOG = Path(__file__).parent
 imf_attributes = _CATALOG / "info-rules-imf.xlsx"
+hello_world_pump = _CATALOG / "hello_world_pump.xlsx"

cognite/neat/_rules/exporters/__init__.py

@@ -7,15 +7,15 @@ from ._rules2yaml import YAMLExporter
 
 __all__ = [
     "BaseExporter",
-    "DMSExporter",
     "CDFExporter",
-    "SemanticDataModelExporter",
-    "OWLExporter",
+    "DMSExporter",
+    "ExcelExporter",
     "GraphExporter",
+    "InstanceTemplateExporter",
+    "OWLExporter",
     "SHACLExporter",
-    "ExcelExporter",
+    "SemanticDataModelExporter",
     "YAMLExporter",
-    "InstanceTemplateExporter",
 ]