cognite-neat 0.98.0__py3-none-any.whl → 0.99.1__py3-none-any.whl

cognite/neat/_rules/importers/_rdf/_imf2rules.py

@@ -0,0 +1,91 @@
+"""This module performs importing of various formats to one of serializations for which
+there are loaders to TransformationRules pydantic class."""
+
+from cognite.neat._rules.importers._rdf._base import BaseRDFImporter
+from cognite.neat._rules.importers._rdf._shared import parse_classes, parse_properties
+
+CLASSES_QUERY = """
+    SELECT ?class_ ?name ?description ?implements
+    WHERE {{
+        VALUES ?type {{ imf:BlockType imf:TerminalType imf:AttributeType }}
+        ?class_ a ?type .
+
+        OPTIONAL {{?class_ rdfs:subClassOf ?parent }}.
+        OPTIONAL {{?class_ rdfs:label|skos:prefLabel ?name }}.
+        OPTIONAL {{?class_ rdfs:comment|skos:definition ?description}}.
+
+
+        # Add imf:Attribute as parent class when no parent is found
+        BIND(IF(!bound(?parent) && ?type = imf:AttributeType, imf:Attribute, ?parent) AS ?implements)
+
+        # FILTERS
+        FILTER (!isBlank(?class_))
+        FILTER (!bound(?implements) || !isBlank(?implements))
+
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "{language}"))
+    }}
+    """
+
+PROPERTIES_QUERY = """
+    SELECT ?class_ ?property_ ?name ?description ?value_type ?min_count ?max_count ?default
+    WHERE
+    {{
+        # CASE 1: Handling Blocks and Terminals
+        {{
+            VALUES ?type {{ imf:BlockType imf:TerminalType }}
+            ?class_ a ?type ;
+                sh:property ?propertyShape .
+                ?propertyShape sh:path ?property_ .
+
+            OPTIONAL {{ ?property_ skos:prefLabel ?name . }}
+            OPTIONAL {{ ?property_ skos:definition ?description . }}
+            OPTIONAL {{ ?property_ rdfs:range ?range . }}
+
+            OPTIONAL {{ ?propertyShape sh:minCount ?min_count . }}
+            OPTIONAL {{ ?propertyShape sh:maxCount ?max_count . }}
+            OPTIONAL {{ ?propertyShape sh:hasValue ?default . }}
+            OPTIONAL {{ ?propertyShape sh:class | sh:qualifiedValueShape/sh:class ?valueShape . }}
+        }}
+
+        UNION
+
+        # CASE 2: Handling Attributes
+        {{
+            ?class_ a imf:AttributeType .
+            BIND(xsd:anyURI AS ?valueShape)
+            BIND(imf:predicate AS ?property_)
+            ?class_  ?property_ ?defaultURI .
+            BIND(STR(?defaultURI) AS ?default)
+
+        }}
+
+        # Set the value type for the property based on sh:class, sh:qualifiedValueType or rdfs:range
+        BIND(IF(BOUND(?valueShape), ?valueShape, IF(BOUND(?range) , ?range , ?valueShape)) AS ?value_type)
+
+        FILTER (!isBlank(?property_))
+        FILTER (!bound(?class_) || !isBlank(?class_))
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "{language}"))
+    }}
+    """
+
+
+class IMFImporter(BaseRDFImporter):
+    """Convert IMF Types provided as SHACL shapes to Input Rules."""
+
+    def _to_rules_components(
+        self,
+    ) -> dict:
+        classes, issue_list = parse_classes(self.graph, CLASSES_QUERY, self.language, self.issue_list)
+        self.issue_list = issue_list
+        properties, issue_list = parse_properties(self.graph, PROPERTIES_QUERY, self.language, self.issue_list)
+        self.issue_list = issue_list
+
+        components = {
+            "Metadata": self._metadata,
+            "Classes": list(classes.values()) if classes else [],
+            "Properties": list(properties.values()) if properties else [],
+        }
+
+        return components

cognite/neat/_rules/importers/_rdf/_inference2rules.py

@@ -1,6 +1,6 @@
 from collections import Counter, defaultdict
 from collections.abc import Mapping
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import ClassVar, cast
 
@@ -8,8 +8,7 @@ from cognite.client import data_modeling as dm
 from rdflib import RDF, Namespace, URIRef
 from rdflib import Literal as RdfLiteral
 
-from cognite.neat._constants import DEFAULT_NAMESPACE
-from cognite.neat._issues.warnings import PropertyValueTypeUndefinedWarning
+from cognite.neat._issues.warnings import PropertySkippedWarning, PropertyValueTypeUndefinedWarning
 from cognite.neat._rules.models import data_types
 from cognite.neat._rules.models.data_types import AnyURI
 from cognite.neat._rules.models.entities._single_value import UnknownEntity
@@ -70,8 +69,15 @@ class InferenceImporter(BaseRDFImporter):
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_INFERENCE_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
         non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
+        language: str = "en",
     ) -> "InferenceImporter":
-        return super().from_graph_store(store, data_model_id, max_number_of_instance, non_existing_node_type)
+        return super().from_graph_store(
+            store,
+            data_model_id,
+            max_number_of_instance,
+            non_existing_node_type,
+            language,
+        )
 
     @classmethod
     def from_file(
@@ -80,8 +86,15 @@ class InferenceImporter(BaseRDFImporter):
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_INFERENCE_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
         non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
+        language: str = "en",
     ) -> "InferenceImporter":
-        return super().from_file(filepath, data_model_id, max_number_of_instance, non_existing_node_type)
+        return super().from_file(
+            filepath,
+            data_model_id,
+            max_number_of_instance,
+            non_existing_node_type,
+            language,
+        )
 
     @classmethod
     def from_json_file(
@@ -89,6 +102,7 @@ class InferenceImporter(BaseRDFImporter):
         filepath: Path,
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_INFERENCE_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
+        language: str = "en",
     ) -> "InferenceImporter":
         raise NotImplementedError("JSON file format is not supported yet.")
 
@@ -98,6 +112,7 @@ class InferenceImporter(BaseRDFImporter):
         filepath: Path,
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_INFERENCE_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
+        language: str = "en",
     ) -> "InferenceImporter":
         raise NotImplementedError("YAML file format is not supported yet.")
 
@@ -156,9 +171,21 @@ class InferenceImporter(BaseRDFImporter):
                     # this is to skip rdf:type property
                     if property_uri == RDF.type:
                         continue
+                    property_id = remove_namespace_from_uri(property_uri)
+                    if property_id in {"external_id", "externalId"}:
+                        skip_issue = PropertySkippedWarning(
+                            resource_type="Property",
+                            identifier=f"{class_id}:{property_id}",
+                            property_name=property_id,
+                            reason="External ID is assumed to be the unique identifier of the instance "
+                            "and is not part of the data model schema.",
+                        )
+                        if skip_issue not in self.issue_list:
+                            self.issue_list.append(skip_issue)
+                        continue
 
                     self._add_uri_namespace_to_prefixes(cast(URIRef, property_uri), prefixes)
-                    property_id = remove_namespace_from_uri(property_uri)
+
                     if isinstance(data_type_uri, URIRef):
                         data_type_uri = self.overwrite_data_types.get(data_type_uri, data_type_uri)
 
@@ -198,11 +225,12 @@ class InferenceImporter(BaseRDFImporter):
 
                     # USE CASE 1: If property is not present in properties
                     if id_ not in properties:
+                        definition["value_type"] = {definition["value_type"]}
                         properties[id_] = definition
 
                     # USE CASE 2: first time redefinition, value type change to multi
                     elif id_ in properties and definition["value_type"] not in properties[id_]["value_type"]:
-                        properties[id_]["value_type"] = properties[id_]["value_type"] + " | " + definition["value_type"]
+                        properties[id_]["value_type"].add(definition["value_type"])
 
                     # USE CASE 3: existing but max count is different
                     elif (
@@ -212,32 +240,12 @@ class InferenceImporter(BaseRDFImporter):
                     ):
                         properties[id_]["max_count"] = max(properties[id_]["max_count"], definition["max_count"])
 
-        # Add comments
-        for id_, property_ in properties.items():
-            if id_ not in count_by_value_type_by_property:
-                continue
-
-            count_by_value_type = count_by_value_type_by_property[id_]
-            count_list = sorted(count_by_value_type.items(), key=lambda item: item[1], reverse=True)
-            # Make the comment more readable by adapting to the number of value types
-            base_string = "<{value_type}> which occurs <{count}> times"
-            if len(count_list) == 1:
-                type_, count = count_list[0]
-                counts_str = f"with value type {base_string.format(value_type=type_, count=count)} in the graph"
-            elif len(count_list) == 2:
-                first = base_string.format(value_type=count_list[0][0], count=count_list[0][1])
-                second = base_string.format(value_type=count_list[1][0], count=count_list[1][1])
-                counts_str = f"with value types {first} and {second} in the graph"
+        # Create multi-value properties otherwise single value
+        for property_ in properties.values():
+            if len(property_["value_type"]) > 1:
+                property_["value_type"] = " | ".join([str(t) for t in property_["value_type"]])
             else:
-                first_part = ", ".join(
-                    base_string.format(value_type=type_, count=count) for type_, count in count_list[:-1]
-                )
-                last = base_string.format(value_type=count_list[-1][0], count=count_list[-1][1])
-                counts_str = f"with value types {first_part} and {last} in the graph"
-
-            class_id = property_["class_"]
-            property_id = property_["property_"]
-            property_["comment"] = f"Class <{class_id}> has property <{property_id}> {counts_str}"
+                property_["value_type"] = next(iter(property_["value_type"]))
 
         return {
             "metadata": self._default_metadata().model_dump(),
@@ -247,14 +255,14 @@ class InferenceImporter(BaseRDFImporter):
         }
 
     def _default_metadata(self):
+        now = datetime.now(timezone.utc)
         return InformationMetadata(
             space=self.data_model_id.space,
             external_id=self.data_model_id.external_id,
             version=self.data_model_id.version,
             name="Inferred Model",
             creator="NEAT",
-            created=datetime.now(),
-            updated=datetime.now(),
+            created=now,
+            updated=now,
             description="Inferred model from knowledge graph",
-            namespace=DEFAULT_NAMESPACE,
         )

cognite/neat/_rules/importers/_rdf/_owl2rules.py

@@ -0,0 +1,80 @@
+"""This module performs importing of various formats to one of serializations for which
+there are loaders to TransformationRules pydantic class."""
+
+from cognite.neat._rules.importers._rdf._base import BaseRDFImporter
+from cognite.neat._rules.importers._rdf._shared import parse_classes, parse_properties
+
+CLASSES_QUERY = """SELECT ?class_ ?name ?description ?implements
+        WHERE {{
+
+        ?class_ a owl:Class .
+        OPTIONAL {{?class_ rdfs:subClassOf ?implements }}.
+        OPTIONAL {{?class_ rdfs:label|skos:prefLabel ?name }}.
+        OPTIONAL {{?class_ rdfs:comment|skos:definition ?description}} .
+
+
+        FILTER (!isBlank(?class_))
+        FILTER (!bound(?implements) || !isBlank(?implements))
+
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "{language}"))
+
+    }}
+    """
+
+PROPERTIES_QUERY = """
+
+    SELECT ?class_ ?property_ ?name ?description ?value_type ?minCount ?maxCount ?default
+    WHERE {{
+        ?property_ a ?property_Type.
+        FILTER (?property_Type IN (owl:ObjectProperty, owl:DatatypeProperty ) )
+        OPTIONAL {{?property_ rdfs:domain ?class_ }}.
+        OPTIONAL {{?property_ rdfs:range ?value_type }}.
+        OPTIONAL {{?property_ rdfs:label|skos:prefLabel ?name }}.
+        OPTIONAL {{?property_ rdfs:comment|skos:definition ?description}}.
+        OPTIONAL {{?property_ owl:maxCardinality ?maxCount}}.
+        OPTIONAL {{?property_ owl:minCardinality ?minCount}}.
+
+        # FILTERS
+        FILTER (!isBlank(?property_))
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "{language}"))
+    }}
+    """
+
+
+class OWLImporter(BaseRDFImporter):
+    """Convert OWL ontology to tables/ transformation rules / Excel file.
+
+        Args:
+            filepath: Path to OWL ontology
+
+    !!! Note
+        OWL Ontologies are information models which completeness varies. As such, constructing functional
+        data model directly will often be impossible, therefore the produced Rules object will be ill formed.
+        To avoid this, neat will automatically attempt to make the imported rules compliant by adding default
+        values for missing information, attaching dangling properties to default containers based on the
+        property type, etc.
+
+        One has to be aware that NEAT will be opinionated about how to make the ontology
+        compliant, and that the resulting rules may not be what you expect.
+
+    """
+
+    def _to_rules_components(
+        self,
+    ) -> dict:
+        classes, issue_list = parse_classes(self.graph, CLASSES_QUERY, self.language, self.issue_list)
+        self.issue_list = issue_list
+
+        # NeatError
+        properties, issue_list = parse_properties(self.graph, PROPERTIES_QUERY, self.language, self.issue_list)
+        self.issue_list = issue_list
+
+        components = {
+            "Metadata": self._metadata,
+            "Classes": list(classes.values()) if classes else [],
+            "Properties": list(properties.values()) if properties else [],
+        }
+
+        return components