cognite-neat 0.77.8__py3-none-any.whl → 0.77.10__py3-none-any.whl

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.77.8"
+__version__ = "0.77.10"

cognite/neat/rules/importers/__init__.py

@@ -1,6 +1,7 @@
 from ._base import BaseImporter
 from ._dms2rules import DMSImporter
 from ._dtdl2rules import DTDLImporter
+from ._inference2rules import InferenceImporter
 from ._owl2rules import OWLImporter
 from ._spreadsheet2rules import ExcelImporter, GoogleSheetImporter
 from ._yaml2rules import YAMLImporter
@@ -13,4 +14,5 @@ __all__ = [
     "GoogleSheetImporter",
     "DTDLImporter",
     "YAMLImporter",
+    "InferenceImporter",
 ]

cognite/neat/rules/importers/_inference2rules.py

@@ -0,0 +1,221 @@
+from datetime import datetime
+from pathlib import Path
+from typing import Literal, cast, overload
+
+from rdflib import Graph, Namespace, URIRef
+from rdflib import Literal as RdfLiteral
+
+import cognite.neat.rules.issues as issues
+from cognite.neat.constants import PREFIXES
+from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
+from cognite.neat.rules.issues import IssueList
+from cognite.neat.rules.models import InformationRules, RoleTypes
+from cognite.neat.rules.models._base import MatchType
+from cognite.neat.rules.models.information import (
+    InformationMetadata,
+    InformationRulesInput,
+)
+from cognite.neat.utils.utils import get_namespace, remove_namespace
+
+ORDERED_CLASSES_QUERY = """SELECT ?class (count(?s) as ?instances )
+                           WHERE { ?s a ?class . }
+                           group by ?class order by DESC(?instances)"""
+
+INSTANCES_OF_CLASS_QUERY = """SELECT ?s WHERE { ?s a <class> . }"""
+
+INSTANCE_PROPERTIES_DEFINITION = """SELECT ?property (count(?property) as ?occurrence) ?dataType ?objectType
+                                    WHERE {<instance_id> ?property ?value .
+                                           BIND(datatype(?value) AS ?dataType)
+                                           OPTIONAL {?value rdf:type ?objectType .}}
+                                    GROUP BY ?property ?dataType ?objectType"""
+
+
+class InferenceImporter(BaseImporter):
+    """Rules inference through analysis of knowledge graph provided in various formats.
+
+    Args:
+        issue_list: Issue list to store issues
+        graph: Knowledge graph
+        max_number_of_instance: Maximum number of instances to be used in inference
+        make_compliant: If True, NEAT will attempt to make the imported rules compliant with CDF
+    """
+
+    def __init__(
+        self, issue_list: IssueList, graph: Graph, max_number_of_instance: int = -1, make_compliant: bool = False
+    ):
+        self.issue_list = issue_list
+        self.graph = graph
+        self.max_number_of_instance = max_number_of_instance
+        self.make_compliant = make_compliant
+
+    @classmethod
+    def from_rdf_file(cls, filepath: Path, make_compliant: bool = False, max_number_of_instance: int = -1):
+        issue_list = IssueList(title=f"'{filepath.name}'")
+
+        graph = Graph()
+        try:
+            graph.parse(filepath)
+        except Exception:
+            issue_list.append(issues.fileread.FileReadError(filepath))
+
+        return cls(issue_list, graph, make_compliant=make_compliant, max_number_of_instance=max_number_of_instance)
+
+    @classmethod
+    def from_json_file(cls, filepath: Path, make_compliant: bool = False, max_number_of_instance: int = -1):
+        raise NotImplementedError("JSON file format is not supported yet.")
+
+    @classmethod
+    def from_yaml_file(cls, filepath: Path, make_compliant: bool = False, max_number_of_instance: int = -1):
+        raise NotImplementedError("YAML file format is not supported yet.")
+
+    @classmethod
+    def from_xml_file(cls, filepath: Path, make_compliant: bool = False, max_number_of_instance: int = -1):
+        raise NotImplementedError("JSON file format is not supported yet.")
+
+    @overload
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
+
+    @overload
+    def to_rules(
+        self,
+        errors: Literal["continue"] = "continue",
+        role: RoleTypes | None = None,
+    ) -> tuple[Rules | None, IssueList]: ...
+
+    def to_rules(
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
+    ) -> tuple[Rules | None, IssueList] | Rules:
+        """
+        Creates `Rules` object from the data for target role.
+        """
+
+        if self.issue_list.has_errors:
+            # In case there were errors during the import, the to_rules method will return None
+            return self._return_or_raise(self.issue_list, errors)
+
+        rules_dict = self._to_rules_components()
+
+        # adding additional prefix
+        rules_dict["prefixes"][rules_dict["metadata"]["prefix"]] = rules_dict["metadata"]["namespace"]
+
+        with _handle_issues(self.issue_list) as future:
+            rules: InformationRules
+            rules = InformationRulesInput.load(rules_dict).as_rules()
+
+        if future.result == "failure" or self.issue_list.has_errors:
+            return self._return_or_raise(self.issue_list, errors)
+
+        if self.make_compliant and rules:
+            rules = self._make_dms_compliant_rules(rules)
+
+        return self._to_output(
+            rules,
+            self.issue_list,
+            errors=errors,
+            role=role,
+        )
+
+    def _to_rules_components(
+        self,
+    ) -> dict:
+        """Convert RDF graph to dictionary defining data model and prefixes of the graph
+
+        Args:
+            graph: RDF graph to be converted to TransformationRules object
+            max_number_of_instance: Max number of instances to be considered for each class
+
+        Returns:
+            Tuple of data model and prefixes of the graph
+        """
+        classes: dict[str, dict] = {}
+        properties: dict[str, dict] = {}
+        prefixes: dict[str, Namespace] = PREFIXES
+
+        # Infers all the classes in the graph
+        for class_uri, no_instances in self.graph.query(ORDERED_CLASSES_QUERY):  # type: ignore[misc]
+            self._add_uri_namespace_to_prefixes(cast(URIRef, class_uri), prefixes)
+
+            if (class_id := remove_namespace(class_uri)) in classes:
+                # handles cases when class id is already present in classes
+                class_id = f"{class_id}_{len(classes)+1}"
+
+            classes[class_id] = {
+                "class_": class_id,
+                "reference": class_uri,
+                "match_type": MatchType.exact,
+                "comment": f"Inferred from knowledge graph, where this class has {no_instances} instances",
+            }
+
+        # Infers all the properties of the class
+        for class_id, class_definition in classes.items():
+            for (instance,) in self.graph.query(  # type: ignore[misc]
+                INSTANCES_OF_CLASS_QUERY.replace("class", class_definition["reference"])
+                if self.max_number_of_instance < 0
+                else INSTANCES_OF_CLASS_QUERY.replace("class", class_definition["reference"])
+                + f" LIMIT {self.max_number_of_instance}"
+            ):
+                for property_uri, occurrence, data_type_uri, object_type_uri in self.graph.query(  # type: ignore[misc]
+                    INSTANCE_PROPERTIES_DEFINITION.replace("instance_id", instance)
+                ):  # type: ignore[misc]
+                    property_id = remove_namespace(property_uri)
+                    self._add_uri_namespace_to_prefixes(cast(URIRef, property_uri), prefixes)
+                    value_type_uri = data_type_uri if data_type_uri else object_type_uri
+
+                    # this is to skip rdf:type property
+                    if not value_type_uri:
+                        continue
+
+                    self._add_uri_namespace_to_prefixes(cast(URIRef, value_type_uri), prefixes)
+                    value_type_id = remove_namespace(value_type_uri)
+                    id_ = f"{class_id}:{property_id}:{value_type_id}"
+
+                    definition = {
+                        "class_": class_id,
+                        "property_": property_id,
+                        "max_occurrence": cast(RdfLiteral, occurrence).value,
+                        "value_type": value_type_id,
+                        "reference": property_uri,
+                    }
+
+                    # USE CASE 1: If property is not present in properties
+                    if id_ not in properties:
+                        properties[id_] = definition
+
+                    # USE CASE 2: If property is present in properties but with different max count
+                    elif id_ in properties and not (properties[id_]["max_count"] == definition["max_count"]):
+                        properties[id_]["max_count"] = max(properties[id_]["max_count"], definition["max_count"])
+
+        return {
+            "metadata": self._default_metadata().model_dump(),
+            "classes": list(classes.values()),
+            "properties": list(properties.values()),
+            "prefixes": prefixes,
+        }
+
+    @classmethod
+    def _add_uri_namespace_to_prefixes(cls, URI: URIRef, prefixes: dict[str, Namespace]):
+        """Add URI to prefixes dict if not already present
+
+        Args:
+            URI: URI from which namespace is being extracted
+            prefixes: Dict of prefixes and namespaces
+        """
+        if Namespace(get_namespace(URI)) not in prefixes.values():
+            prefixes[f"prefix-{len(prefixes)+1}"] = Namespace(get_namespace(URI))
+
+    @classmethod
+    def _default_metadata(cls):
+        return InformationMetadata(
+            name="Inferred Model",
+            creator="NEAT",
+            version="inferred",
+            created=datetime.now(),
+            updated=datetime.now(),
+            description="Inferred model from knowledge graph",
+            prefix="inferred",
+            namespace="http://purl.org/cognite/neat/inferred/",
+        )
+
+    @classmethod
+    def _make_dms_compliant_rules(cls, rules: InformationRules) -> InformationRules:
+        raise NotImplementedError("Compliance with DMS is not supported yet.")

cognite/neat/rules/importers/_owl2rules/_owl2rules.py

@@ -1,8 +1,6 @@
 """This module performs importing of various formats to one of serializations for which
 there are loaders to TransformationRules pydantic class."""
 
-# TODO: if this module grows too big, split it into several files and place under ./converter directory
-
 from pathlib import Path
 from typing import Literal, overload
 
@@ -22,7 +20,8 @@ class OWLImporter(BaseImporter):
     """Convert OWL ontology to tables/ transformation rules / Excel file.
 
         Args:
-            owl_filepath: Path to OWL ontology
+            filepath: Path to OWL ontology
+            make_compliant: If True, NEAT will attempt to make the imported rules compliant with CDF
 
     !!! Note
         OWL Ontologies typically lacks some information that is required for making a complete

{cognite_neat-0.77.8.dist-info → cognite_neat-0.77.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.77.8
+Version: 0.77.10
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0
@@ -50,7 +50,7 @@ Requires-Dist: schedule (>=1,<2)
 Requires-Dist: tomli (>=2.0.1,<3.0.0) ; python_version < "3.11"
 Requires-Dist: typing_extensions (>=4.8,<5.0) ; python_version < "3.11"
 Requires-Dist: urllib3 (>=2,<3)
-Requires-Dist: uvicorn[standard] (>=0.29.0,<0.30.0)
+Requires-Dist: uvicorn[standard] (>=0,<1)
 Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
 Project-URL: Repository, https://github.com/cognitedata/neat
 Description-Content-Type: text/markdown

{cognite_neat-0.77.8.dist-info → cognite_neat-0.77.10.dist-info}/RECORD

@@ -1,5 +1,5 @@
 cognite/neat/__init__.py,sha256=v-rRiDOgZ3sQSMQKq0vgUQZvpeOkoHFXissAx6Ktg84,61
-cognite/neat/_version.py,sha256=mXpwNIS8gqWNHN489kDHmqkjGuUNEJ_NwDA07_xdNQk,23
+cognite/neat/_version.py,sha256=-swOEBp1OmcqYEuvtO9gV117lIY0GkQSzACgRA9L1AE,24
 cognite/neat/app/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/app/api/asgi/metrics.py,sha256=nxFy7L5cChTI0a-zkCiJ59Aq8yLuIJp5c9Dg0wRXtV0,152
 cognite/neat/app/api/configuration.py,sha256=2U5M6M252swvQPQyooA1EBzFUZNtcTmuSaywfJDgckM,4232
@@ -170,7 +170,7 @@ cognite/neat/rules/exporters/_rules2excel.py,sha256=HvUdXYHxfLMijYWdTnfqCsw3Izf8
 cognite/neat/rules/exporters/_rules2ontology.py,sha256=NWS3cn2927LQqW_PdQ-92OLIlmIKGNk7xh5yOMAyj94,20120
 cognite/neat/rules/exporters/_rules2yaml.py,sha256=GA8eUYRxUfIU6IMvlyGO5JidkOD5eUKSbH3qAiFiaCg,3026
 cognite/neat/rules/exporters/_validation.py,sha256=OlKIyf4nhSDehJwFHDQ8Zdf6HpNfW7dSe2s67eywHu4,4078
-cognite/neat/rules/importers/__init__.py,sha256=zqNbGpvdVhYkLjWx1i9dJ3FXzYGtuQyTydUYsj-BndQ,408
+cognite/neat/rules/importers/__init__.py,sha256=gR6_TAEa3iO5NCLKRztHg-FMiLdBnx47Z3iSzbwLfcE,481
 cognite/neat/rules/importers/_base.py,sha256=GUiJrYwJ25thI71iS9hCeP_iSZ0Vv8ou3z6MfD07FAk,4274
 cognite/neat/rules/importers/_dms2rules.py,sha256=5yJGYkM7lAMu-QfO0_r59WE4RGtMu2smMqLm16ohgLQ,18994
 cognite/neat/rules/importers/_dtdl2rules/__init__.py,sha256=CNR-sUihs2mnR1bPMKs3j3L4ds3vFTsrl6YycExZTfU,68
@@ -178,11 +178,12 @@ cognite/neat/rules/importers/_dtdl2rules/_unit_lookup.py,sha256=wW4saKva61Q_i17g
 cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py,sha256=ysmWUxZ0npwrTB0uiH5jA0v37sfCwowGaYk17IyxPUU,12663
 cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py,sha256=QDyGt5YBaxzF4v_oCFSgKRSpwVdVruDU3-VW0DEiHbY,6718
 cognite/neat/rules/importers/_dtdl2rules/spec.py,sha256=tim_MfN1J0F3Oeqk3BMgIA82d_MZvhRuRMsLK3B4PYc,11897
+cognite/neat/rules/importers/_inference2rules.py,sha256=VYs4SCYQk3OMJnCkpJ-fTsK73OfK8cOyYXBAdYI3_WU,9389
 cognite/neat/rules/importers/_owl2rules/__init__.py,sha256=tdGcrgtozdQyST-pTlxIa4cLBNTLvtk1nNYR4vOdFSw,63
 cognite/neat/rules/importers/_owl2rules/_owl2classes.py,sha256=LInFeBq-NbBIuMEAwgWch2a4DbBUt4_OMqPkwehW-sw,7591
 cognite/neat/rules/importers/_owl2rules/_owl2metadata.py,sha256=NdPN0dBB0NYkAcfC0yrYdIrGfdPbl5gfeGnSV3EtUPM,7786
 cognite/neat/rules/importers/_owl2rules/_owl2properties.py,sha256=BLptGmH-Aa5gZu0hDIxSZTrn9GmB2FicWgRYoETLSnQ,7437
-cognite/neat/rules/importers/_owl2rules/_owl2rules.py,sha256=H2Vv56hXGFnq_b0obWGWr5ErDFcoWpT8G2uy89100cU,6925
+cognite/neat/rules/importers/_owl2rules/_owl2rules.py,sha256=vyXtAhHCg-zCv_sfq28bx5BK96GXoCB4KA5LeO_ZuvU,6917
 cognite/neat/rules/importers/_spreadsheet2rules.py,sha256=nKSJyZGoTho0bqQ_5_1XB9Z1C-MwovRgkVrC-AhOuzs,12438
 cognite/neat/rules/importers/_yaml2rules.py,sha256=F0uksSz1A3po5OlRM2152_w5j8D9oYTLB9NFTkSMlWI,4275
 cognite/neat/rules/issues/__init__.py,sha256=Ms6jgCxCezc5IgTOwCFtXQPtoVFfOvdcXj84_rs917I,563
@@ -275,8 +276,8 @@ cognite/neat/workflows/steps_registry.py,sha256=fkTX14ZA7_gkUYfWIlx7A1XbCidvqR23
 cognite/neat/workflows/tasks.py,sha256=dqlJwKAb0jlkl7abbY8RRz3m7MT4SK8-7cntMWkOYjw,788
 cognite/neat/workflows/triggers.py,sha256=_BLNplzoz0iic367u1mhHMHiUrCwP-SLK6_CZzfODX0,7071
 cognite/neat/workflows/utils.py,sha256=gKdy3RLG7ctRhbCRwaDIWpL9Mi98zm56-d4jfHDqP1E,453
-cognite_neat-0.77.8.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-0.77.8.dist-info/METADATA,sha256=fr3l3w0YmPmZwtA9G4rKyL5fVBvN8ThIeBQNtJtzLwk,9316
-cognite_neat-0.77.8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cognite_neat-0.77.8.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
-cognite_neat-0.77.8.dist-info/RECORD,,
+cognite_neat-0.77.10.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-0.77.10.dist-info/METADATA,sha256=ioQ2hIZ3OVqhuHAAeHK1WePxy_c5Hu_n231b-m8E2R4,9307
+cognite_neat-0.77.10.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cognite_neat-0.77.10.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
+cognite_neat-0.77.10.dist-info/RECORD,,