cognite-neat 0.99.0__py3-none-any.whl → 0.99.1__py3-none-any.whl

cognite/neat/_rules/importers/_rdf/_base.py

@@ -1,13 +1,16 @@
+from datetime import datetime
 from pathlib import Path
 
 from cognite.client import data_modeling as dm
-from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SH, SKOS, XSD, Graph, Namespace, URIRef
+from rdflib import Graph, Namespace, URIRef
 
+from cognite.neat._constants import get_default_prefixes
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import FileReadError
 from cognite.neat._issues.errors._general import NeatValueError
 from cognite.neat._rules._shared import ReadRules
 from cognite.neat._rules.importers._base import BaseImporter
+from cognite.neat._rules.models._base_rules import RoleTypes
 from cognite.neat._rules.models.data_types import AnyURI
 from cognite.neat._rules.models.entities import UnknownEntity
 from cognite.neat._rules.models.information import (
@@ -30,6 +33,13 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
         graph: Knowledge graph
         data_model_id: Data model id to be used for the imported rules
         space: CDF Space to be used for the imported rules
+        language: Language for description and human readable entity names
+
+
+
+    !!! note "Language"
+        Language is provided as ISO 639-1 code. If not provided, English will be used as default.
+
     """
 
     def __init__(
@@ -39,6 +49,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
         data_model_id: dm.DataModelId | tuple[str, str, str],
         max_number_of_instance: int,
         non_existing_node_type: UnknownEntity | AnyURI,
+        language: str,
     ) -> None:
         self.issue_list = issue_list
         self.graph = graph
@@ -48,6 +59,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
 
         self.max_number_of_instance = max_number_of_instance
         self.non_existing_node_type = non_existing_node_type
+        self.language = language
 
     @classmethod
     def from_graph_store(
@@ -56,6 +68,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_RDF_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
         non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
+        language: str = "en",
     ):
         return cls(
             IssueList(title=f"{cls.__name__} issues"),
@@ -63,6 +76,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
             data_model_id=data_model_id,
             max_number_of_instance=max_number_of_instance,
             non_existing_node_type=non_existing_node_type,
+            language=language,
         )
 
     @classmethod
@@ -72,6 +86,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_RDF_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
         non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
+        language: str = "en",
     ):
         issue_list = IssueList(title=f"{cls.__name__} issues")
 
@@ -82,15 +97,8 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
             issue_list.append(FileReadError(filepath, str(e)))
 
         # bind key namespaces
-        graph.bind("owl", OWL)
-        graph.bind("rdf", RDF)
-        graph.bind("rdfs", RDFS)
-        graph.bind("dcterms", DCTERMS)
-        graph.bind("dc", DC)
-        graph.bind("skos", SKOS)
-        graph.bind("sh", SH)
-        graph.bind("xsd", XSD)
-        graph.bind("imf", "http://ns.imfid.org/imf#")
+        for prefix, namespace in get_default_prefixes().items():
+            graph.bind(prefix, namespace)
 
         return cls(
             issue_list,
@@ -98,6 +106,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
             data_model_id=data_model_id,
             max_number_of_instance=max_number_of_instance,
             non_existing_node_type=non_existing_node_type,
+            language=language,
         )
 
     def to_rules(
@@ -129,4 +138,18 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
             prefixes: Dict of prefixes and namespaces
         """
         if Namespace(get_namespace(URI)) not in prefixes.values():
-            prefixes[f"prefix-{len(prefixes)+1}"] = Namespace(get_namespace(URI))
+            prefixes[f"prefix_{len(prefixes)+1}"] = Namespace(get_namespace(URI))
+
+    @property
+    def _metadata(self) -> dict:
+        return {
+            "role": RoleTypes.information,
+            "space": self.data_model_id.space,
+            "external_id": self.data_model_id.external_id,
+            "version": self.data_model_id.version,
+            "created": datetime.now().replace(microsecond=0),
+            "updated": datetime.now().replace(microsecond=0),
+            "name": None,
+            "description": f"Data model imported using {type(self).__name__}",
+            "creator": "Neat",
+        }

cognite/neat/_rules/importers/_rdf/_imf2rules.py

@@ -0,0 +1,91 @@
+"""This module performs importing of various formats to one of serializations for which
+there are loaders to TransformationRules pydantic class."""
+
+from cognite.neat._rules.importers._rdf._base import BaseRDFImporter
+from cognite.neat._rules.importers._rdf._shared import parse_classes, parse_properties
+
+CLASSES_QUERY = """
+    SELECT ?class_ ?name ?description ?implements
+    WHERE {{
+        VALUES ?type {{ imf:BlockType imf:TerminalType imf:AttributeType }}
+        ?class_ a ?type .
+
+        OPTIONAL {{?class_ rdfs:subClassOf ?parent }}.
+        OPTIONAL {{?class_ rdfs:label|skos:prefLabel ?name }}.
+        OPTIONAL {{?class_ rdfs:comment|skos:definition ?description}}.
+
+
+        # Add imf:Attribute as parent class when no parent is found
+        BIND(IF(!bound(?parent) && ?type = imf:AttributeType, imf:Attribute, ?parent) AS ?implements)
+
+        # FILTERS
+        FILTER (!isBlank(?class_))
+        FILTER (!bound(?implements) || !isBlank(?implements))
+
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "{language}"))
+    }}
+    """
+
+PROPERTIES_QUERY = """
+    SELECT ?class_ ?property_ ?name ?description ?value_type ?min_count ?max_count ?default
+    WHERE
+    {{
+        # CASE 1: Handling Blocks and Terminals
+        {{
+            VALUES ?type {{ imf:BlockType imf:TerminalType }}
+            ?class_ a ?type ;
+                sh:property ?propertyShape .
+                ?propertyShape sh:path ?property_ .
+
+            OPTIONAL {{ ?property_ skos:prefLabel ?name . }}
+            OPTIONAL {{ ?property_ skos:definition ?description . }}
+            OPTIONAL {{ ?property_ rdfs:range ?range . }}
+
+            OPTIONAL {{ ?propertyShape sh:minCount ?min_count . }}
+            OPTIONAL {{ ?propertyShape sh:maxCount ?max_count . }}
+            OPTIONAL {{ ?propertyShape sh:hasValue ?default . }}
+            OPTIONAL {{ ?propertyShape sh:class | sh:qualifiedValueShape/sh:class ?valueShape . }}
+        }}
+
+        UNION
+
+        # CASE 2: Handling Attributes
+        {{
+            ?class_ a imf:AttributeType .
+            BIND(xsd:anyURI AS ?valueShape)
+            BIND(imf:predicate AS ?property_)
+            ?class_  ?property_ ?defaultURI .
+            BIND(STR(?defaultURI) AS ?default)
+
+        }}
+
+        # Set the value type for the property based on sh:class, sh:qualifiedValueType or rdfs:range
+        BIND(IF(BOUND(?valueShape), ?valueShape, IF(BOUND(?range) , ?range , ?valueShape)) AS ?value_type)
+
+        FILTER (!isBlank(?property_))
+        FILTER (!bound(?class_) || !isBlank(?class_))
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "{language}"))
+    }}
+    """
+
+
+class IMFImporter(BaseRDFImporter):
+    """Convert IMF Types provided as SHACL shapes to Input Rules."""
+
+    def _to_rules_components(
+        self,
+    ) -> dict:
+        classes, issue_list = parse_classes(self.graph, CLASSES_QUERY, self.language, self.issue_list)
+        self.issue_list = issue_list
+        properties, issue_list = parse_properties(self.graph, PROPERTIES_QUERY, self.language, self.issue_list)
+        self.issue_list = issue_list
+
+        components = {
+            "Metadata": self._metadata,
+            "Classes": list(classes.values()) if classes else [],
+            "Properties": list(properties.values()) if properties else [],
+        }
+
+        return components

cognite/neat/_rules/importers/_rdf/_inference2rules.py

@@ -69,8 +69,15 @@ class InferenceImporter(BaseRDFImporter):
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_INFERENCE_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
         non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
+        language: str = "en",
     ) -> "InferenceImporter":
-        return super().from_graph_store(store, data_model_id, max_number_of_instance, non_existing_node_type)
+        return super().from_graph_store(
+            store,
+            data_model_id,
+            max_number_of_instance,
+            non_existing_node_type,
+            language,
+        )
 
     @classmethod
     def from_file(
@@ -79,8 +86,15 @@ class InferenceImporter(BaseRDFImporter):
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_INFERENCE_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
         non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
+        language: str = "en",
     ) -> "InferenceImporter":
-        return super().from_file(filepath, data_model_id, max_number_of_instance, non_existing_node_type)
+        return super().from_file(
+            filepath,
+            data_model_id,
+            max_number_of_instance,
+            non_existing_node_type,
+            language,
+        )
 
     @classmethod
     def from_json_file(
@@ -88,6 +102,7 @@ class InferenceImporter(BaseRDFImporter):
         filepath: Path,
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_INFERENCE_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
+        language: str = "en",
     ) -> "InferenceImporter":
         raise NotImplementedError("JSON file format is not supported yet.")
 
@@ -97,6 +112,7 @@ class InferenceImporter(BaseRDFImporter):
         filepath: Path,
         data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_INFERENCE_DATA_MODEL_ID,
         max_number_of_instance: int = -1,
+        language: str = "en",
     ) -> "InferenceImporter":
         raise NotImplementedError("YAML file format is not supported yet.")
 

cognite/neat/_rules/importers/_rdf/_owl2rules.py

@@ -0,0 +1,80 @@
+"""This module performs importing of various formats to one of serializations for which
+there are loaders to TransformationRules pydantic class."""
+
+from cognite.neat._rules.importers._rdf._base import BaseRDFImporter
+from cognite.neat._rules.importers._rdf._shared import parse_classes, parse_properties
+
+CLASSES_QUERY = """SELECT ?class_ ?name ?description ?implements
+        WHERE {{
+
+        ?class_ a owl:Class .
+        OPTIONAL {{?class_ rdfs:subClassOf ?implements }}.
+        OPTIONAL {{?class_ rdfs:label|skos:prefLabel ?name }}.
+        OPTIONAL {{?class_ rdfs:comment|skos:definition ?description}} .
+
+
+        FILTER (!isBlank(?class_))
+        FILTER (!bound(?implements) || !isBlank(?implements))
+
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "{language}"))
+
+    }}
+    """
+
+PROPERTIES_QUERY = """
+
+    SELECT ?class_ ?property_ ?name ?description ?value_type ?minCount ?maxCount ?default
+    WHERE {{
+        ?property_ a ?property_Type.
+        FILTER (?property_Type IN (owl:ObjectProperty, owl:DatatypeProperty ) )
+        OPTIONAL {{?property_ rdfs:domain ?class_ }}.
+        OPTIONAL {{?property_ rdfs:range ?value_type }}.
+        OPTIONAL {{?property_ rdfs:label|skos:prefLabel ?name }}.
+        OPTIONAL {{?property_ rdfs:comment|skos:definition ?description}}.
+        OPTIONAL {{?property_ owl:maxCardinality ?maxCount}}.
+        OPTIONAL {{?property_ owl:minCardinality ?minCount}}.
+
+        # FILTERS
+        FILTER (!isBlank(?property_))
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "{language}"))
+    }}
+    """
+
+
+class OWLImporter(BaseRDFImporter):
+    """Convert OWL ontology to tables/ transformation rules / Excel file.
+
+        Args:
+            filepath: Path to OWL ontology
+
+    !!! Note
+        OWL Ontologies are information models which completeness varies. As such, constructing functional
+        data model directly will often be impossible, therefore the produced Rules object will be ill formed.
+        To avoid this, neat will automatically attempt to make the imported rules compliant by adding default
+        values for missing information, attaching dangling properties to default containers based on the
+        property type, etc.
+
+        One has to be aware that NEAT will be opinionated about how to make the ontology
+        compliant, and that the resulting rules may not be what you expect.
+
+    """
+
+    def _to_rules_components(
+        self,
+    ) -> dict:
+        classes, issue_list = parse_classes(self.graph, CLASSES_QUERY, self.language, self.issue_list)
+        self.issue_list = issue_list
+
+        # NeatError
+        properties, issue_list = parse_properties(self.graph, PROPERTIES_QUERY, self.language, self.issue_list)
+        self.issue_list = issue_list
+
+        components = {
+            "Metadata": self._metadata,
+            "Classes": list(classes.values()) if classes else [],
+            "Properties": list(properties.values()) if properties else [],
+        }
+
+        return components