cognite-neat 0.99.0__py3-none-any.whl → 0.100.0__py3-none-any.whl

cognite/neat/_rules/importers/_rdf/_base.py

@@ -1,13 +1,16 @@
+from datetime import datetime
 from pathlib import Path
 
 from cognite.client import data_modeling as dm
-from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SH, SKOS, XSD, Graph, Namespace, URIRef
+from rdflib import Graph, Namespace, URIRef
 
+from cognite.neat._constants import get_default_prefixes
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import FileReadError
 from cognite.neat._issues.errors._general import NeatValueError
 from cognite.neat._rules._shared import ReadRules
 from cognite.neat._rules.importers._base import BaseImporter
+from cognite.neat._rules.models._base_rules import RoleTypes
 from cognite.neat._rules.models.data_types import AnyURI
 from cognite.neat._rules.models.entities import UnknownEntity
 from cognite.neat._rules.models.information import (
@@ -30,6 +33,13 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
         graph: Knowledge graph
         data_model_id: Data model id to be used for the imported rules
         space: CDF Space to be used for the imported rules
+        language: Language for description and human readable entity names
+
+
+
+    !!! note "Language"
+        Language is provided as ISO 639-1 code. If not provided, English will be used as default.
+
     """
 
     def __init__(
@@ -39,6 +49,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
         data_model_id: dm.DataModelId | tuple[str, str, str],
         max_number_of_instance: int,
         non_existing_node_type: UnknownEntity | AnyURI,
+        language: str,
     ) -> None:
         self.issue_list = issue_list
         self.graph = graph
@@ -48,6 +59,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
 
         self.max_number_of_instance = max_number_of_instance
         self.non_existing_node_type = non_existing_node_type
+        self.language = language
 
     @classmethod
     def from_graph_store(
@@ -56,6 +68,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_RDF_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
         non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
+        language: str = "en",
     ):
         return cls(
             IssueList(title=f"{cls.__name__} issues"),
@@ -63,6 +76,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
             data_model_id=data_model_id,
             max_number_of_instance=max_number_of_instance,
             non_existing_node_type=non_existing_node_type,
+            language=language,
         )
 
     @classmethod
@@ -72,6 +86,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_RDF_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
         non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
+        language: str = "en",
     ):
         issue_list = IssueList(title=f"{cls.__name__} issues")
 
@@ -82,15 +97,8 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
             issue_list.append(FileReadError(filepath, str(e)))
 
         # bind key namespaces
-        graph.bind("owl", OWL)
-        graph.bind("rdf", RDF)
-        graph.bind("rdfs", RDFS)
-        graph.bind("dcterms", DCTERMS)
-        graph.bind("dc", DC)
-        graph.bind("skos", SKOS)
-        graph.bind("sh", SH)
-        graph.bind("xsd", XSD)
-        graph.bind("imf", "http://ns.imfid.org/imf#")
+        for prefix, namespace in get_default_prefixes().items():
+            graph.bind(prefix, namespace)
 
         return cls(
             issue_list,
@@ -98,6 +106,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
             data_model_id=data_model_id,
             max_number_of_instance=max_number_of_instance,
             non_existing_node_type=non_existing_node_type,
+            language=language,
         )
 
     def to_rules(
@@ -129,4 +138,18 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
             prefixes: Dict of prefixes and namespaces
         """
         if Namespace(get_namespace(URI)) not in prefixes.values():
-            prefixes[f"prefix-{len(prefixes)+1}"] = Namespace(get_namespace(URI))
+            prefixes[f"prefix_{len(prefixes)+1}"] = Namespace(get_namespace(URI))
+
+    @property
+    def _metadata(self) -> dict:
+        return {
+            "role": RoleTypes.information,
+            "space": self.data_model_id.space,
+            "external_id": self.data_model_id.external_id,
+            "version": self.data_model_id.version,
+            "created": datetime.now().replace(microsecond=0),
+            "updated": datetime.now().replace(microsecond=0),
+            "name": None,
+            "description": f"Data model imported using {type(self).__name__}",
+            "creator": "Neat",
+        }

cognite/neat/_rules/importers/_rdf/_imf2rules.py

@@ -0,0 +1,91 @@
+"""This module performs importing of various formats to one of serializations for which
+there are loaders to TransformationRules pydantic class."""
+
+from cognite.neat._rules.importers._rdf._base import BaseRDFImporter
+from cognite.neat._rules.importers._rdf._shared import parse_classes, parse_properties
+
+CLASSES_QUERY = """
+    SELECT ?class_ ?name ?description ?implements
+    WHERE {{
+        VALUES ?type {{ imf:BlockType imf:TerminalType imf:AttributeType }}
+        ?class_ a ?type .
+
+        OPTIONAL {{?class_ rdfs:subClassOf ?parent }}.
+        OPTIONAL {{?class_ rdfs:label|skos:prefLabel ?name }}.
+        OPTIONAL {{?class_ rdfs:comment|skos:definition ?description}}.
+
+
+        # Add imf:Attribute as parent class when no parent is found
+        BIND(IF(!bound(?parent) && ?type = imf:AttributeType, imf:Attribute, ?parent) AS ?implements)
+
+        # FILTERS
+        FILTER (!isBlank(?class_))
+        FILTER (!bound(?implements) || !isBlank(?implements))
+
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "{language}"))
+    }}
+    """
+
+PROPERTIES_QUERY = """
+    SELECT ?class_ ?property_ ?name ?description ?value_type ?min_count ?max_count ?default
+    WHERE
+    {{
+        # CASE 1: Handling Blocks and Terminals
+        {{
+            VALUES ?type {{ imf:BlockType imf:TerminalType }}
+            ?class_ a ?type ;
+                sh:property ?propertyShape .
+                ?propertyShape sh:path ?property_ .
+
+            OPTIONAL {{ ?property_ skos:prefLabel ?name . }}
+            OPTIONAL {{ ?property_ skos:definition ?description . }}
+            OPTIONAL {{ ?property_ rdfs:range ?range . }}
+
+            OPTIONAL {{ ?propertyShape sh:minCount ?min_count . }}
+            OPTIONAL {{ ?propertyShape sh:maxCount ?max_count . }}
+            OPTIONAL {{ ?propertyShape sh:hasValue ?default . }}
+            OPTIONAL {{ ?propertyShape sh:class | sh:qualifiedValueShape/sh:class ?valueShape . }}
+        }}
+
+        UNION
+
+        # CASE 2: Handling Attributes
+        {{
+            ?class_ a imf:AttributeType .
+            BIND(xsd:anyURI AS ?valueShape)
+            BIND(imf:predicate AS ?property_)
+            ?class_  ?property_ ?defaultURI .
+            BIND(STR(?defaultURI) AS ?default)
+
+        }}
+
+        # Set the value type for the property based on sh:class, sh:qualifiedValueType or rdfs:range
+        BIND(IF(BOUND(?valueShape), ?valueShape, IF(BOUND(?range) , ?range , ?valueShape)) AS ?value_type)
+
+        FILTER (!isBlank(?property_))
+        FILTER (!bound(?class_) || !isBlank(?class_))
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "{language}"))
+    }}
+    """
+
+
+class IMFImporter(BaseRDFImporter):
+    """Convert IMF Types provided as SHACL shapes to Input Rules."""
+
+    def _to_rules_components(
+        self,
+    ) -> dict:
+        classes, issue_list = parse_classes(self.graph, CLASSES_QUERY, self.language, self.issue_list)
+        self.issue_list = issue_list
+        properties, issue_list = parse_properties(self.graph, PROPERTIES_QUERY, self.language, self.issue_list)
+        self.issue_list = issue_list
+
+        components = {
+            "Metadata": self._metadata,
+            "Classes": list(classes.values()) if classes else [],
+            "Properties": list(properties.values()) if properties else [],
+        }
+
+        return components

cognite/neat/_rules/importers/_rdf/_inference2rules.py

@@ -26,7 +26,14 @@ ORDERED_CLASSES_QUERY = """SELECT ?class (count(?s) as ?instances )
                            WHERE { ?s a ?class . }
                            group by ?class order by DESC(?instances)"""
 
-INSTANCES_OF_CLASS_QUERY = """SELECT ?s WHERE { ?s a <class> . }"""
+
+INSTANCES_OF_CLASS_QUERY = """SELECT ?s ?propertyCount WHERE { ?s a <class> . BIND ('Unknown' as ?propertyCount) }"""
+
+
+INSTANCES_OF_CLASS_RICHNESS_ORDERED_QUERY = """SELECT ?s (COUNT(?p) as ?propertyCount)
+                                               WHERE { ?s a <class> ; ?p ?o . }
+                                               GROUP BY ?s
+                                               ORDER BY DESC(?propertyCount)"""
 
 INSTANCE_PROPERTIES_DEFINITION = """SELECT ?property (count(?property) as ?occurrence) ?dataType ?objectType
                                     WHERE {<instance_id> ?property ?value .
@@ -69,8 +76,15 @@ class InferenceImporter(BaseRDFImporter):
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_INFERENCE_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
         non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
+        language: str = "en",
     ) -> "InferenceImporter":
-        return super().from_graph_store(store, data_model_id, max_number_of_instance, non_existing_node_type)
+        return super().from_graph_store(
+            store,
+            data_model_id,
+            max_number_of_instance,
+            non_existing_node_type,
+            language,
+        )
 
     @classmethod
     def from_file(
@@ -79,8 +93,15 @@ class InferenceImporter(BaseRDFImporter):
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_INFERENCE_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
         non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
+        language: str = "en",
     ) -> "InferenceImporter":
-        return super().from_file(filepath, data_model_id, max_number_of_instance, non_existing_node_type)
+        return super().from_file(
+            filepath,
+            data_model_id,
+            max_number_of_instance,
+            non_existing_node_type,
+            language,
+        )
 
     @classmethod
     def from_json_file(
@@ -88,6 +109,7 @@ class InferenceImporter(BaseRDFImporter):
         filepath: Path,
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_INFERENCE_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
+        language: str = "en",
     ) -> "InferenceImporter":
         raise NotImplementedError("JSON file format is not supported yet.")
 
@@ -97,6 +119,7 @@ class InferenceImporter(BaseRDFImporter):
         filepath: Path,
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_INFERENCE_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
+        language: str = "en",
     ) -> "InferenceImporter":
         raise NotImplementedError("YAML file format is not supported yet.")
 
@@ -141,13 +164,19 @@ class InferenceImporter(BaseRDFImporter):
 
             self._add_uri_namespace_to_prefixes(cast(URIRef, class_uri), prefixes)
 
+        instances_query = (
+            INSTANCES_OF_CLASS_QUERY if self.max_number_of_instance == -1 else INSTANCES_OF_CLASS_RICHNESS_ORDERED_QUERY
+        )
+
         # Infers all the properties of the class
         for class_id, class_definition in classes.items():
-            for (instance,) in self.graph.query(  # type: ignore[misc]
-                INSTANCES_OF_CLASS_QUERY.replace("class", class_definition["uri"])
+            for (  # type: ignore[misc]
+                instance,
+                _,
+            ) in self.graph.query(  # type: ignore[misc]
+                instances_query.replace("class", class_definition["uri"])
                 if self.max_number_of_instance < 0
-                else INSTANCES_OF_CLASS_QUERY.replace("class", class_definition["uri"])
-                + f" LIMIT {self.max_number_of_instance}"
+                else instances_query.replace("class", class_definition["uri"]) + f" LIMIT {self.max_number_of_instance}"
             ):
                 for property_uri, occurrence, data_type_uri, object_type_uri in self.graph.query(  # type: ignore[misc]
                     INSTANCE_PROPERTIES_DEFINITION.replace("instance_id", instance)
@@ -226,6 +255,10 @@ class InferenceImporter(BaseRDFImporter):
 
         # Create multi-value properties otherwise single value
         for property_ in properties.values():
+            # Removes non-existing node type from value type prior final conversion to string
+            if len(property_["value_type"]) > 1 and str(self.non_existing_node_type) in property_["value_type"]:
+                property_["value_type"].remove(str(self.non_existing_node_type))
+
             if len(property_["value_type"]) > 1:
                 property_["value_type"] = " | ".join([str(t) for t in property_["value_type"]])
             else:

cognite/neat/_rules/importers/_rdf/_owl2rules.py

@@ -0,0 +1,80 @@
+"""This module performs importing of various formats to one of serializations for which
+there are loaders to TransformationRules pydantic class."""
+
+from cognite.neat._rules.importers._rdf._base import BaseRDFImporter
+from cognite.neat._rules.importers._rdf._shared import parse_classes, parse_properties
+
+CLASSES_QUERY = """SELECT ?class_ ?name ?description ?implements
+        WHERE {{
+
+        ?class_ a owl:Class .
+        OPTIONAL {{?class_ rdfs:subClassOf ?implements }}.
+        OPTIONAL {{?class_ rdfs:label|skos:prefLabel ?name }}.
+        OPTIONAL {{?class_ rdfs:comment|skos:definition ?description}} .
+
+
+        FILTER (!isBlank(?class_))
+        FILTER (!bound(?implements) || !isBlank(?implements))
+
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "{language}"))
+
+    }}
+    """
+
+PROPERTIES_QUERY = """
+
+    SELECT ?class_ ?property_ ?name ?description ?value_type ?minCount ?maxCount ?default
+    WHERE {{
+        ?property_ a ?property_Type.
+        FILTER (?property_Type IN (owl:ObjectProperty, owl:DatatypeProperty ) )
+        OPTIONAL {{?property_ rdfs:domain ?class_ }}.
+        OPTIONAL {{?property_ rdfs:range ?value_type }}.
+        OPTIONAL {{?property_ rdfs:label|skos:prefLabel ?name }}.
+        OPTIONAL {{?property_ rdfs:comment|skos:definition ?description}}.
+        OPTIONAL {{?property_ owl:maxCardinality ?maxCount}}.
+        OPTIONAL {{?property_ owl:minCardinality ?minCount}}.
+
+        # FILTERS
+        FILTER (!isBlank(?property_))
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "{language}"))
+    }}
+    """
+
+
+class OWLImporter(BaseRDFImporter):
+    """Convert OWL ontology to tables/ transformation rules / Excel file.
+
+        Args:
+            filepath: Path to OWL ontology
+
+    !!! Note
+        OWL Ontologies are information models which completeness varies. As such, constructing functional
+        data model directly will often be impossible, therefore the produced Rules object will be ill formed.
+        To avoid this, neat will automatically attempt to make the imported rules compliant by adding default
+        values for missing information, attaching dangling properties to default containers based on the
+        property type, etc.
+
+        One has to be aware that NEAT will be opinionated about how to make the ontology
+        compliant, and that the resulting rules may not be what you expect.
+
+    """
+
+    def _to_rules_components(
+        self,
+    ) -> dict:
+        classes, issue_list = parse_classes(self.graph, CLASSES_QUERY, self.language, self.issue_list)
+        self.issue_list = issue_list
+
+        # NeatError
+        properties, issue_list = parse_properties(self.graph, PROPERTIES_QUERY, self.language, self.issue_list)
+        self.issue_list = issue_list
+
+        components = {
+            "Metadata": self._metadata,
+            "Classes": list(classes.values()) if classes else [],
+            "Properties": list(properties.values()) if properties else [],
+        }
+
+        return components