cognite-neat 0.88.3__py3-none-any.whl → 0.90.0__py3-none-any.whl

cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py

@@ -37,16 +37,20 @@ def parse_imf_to_classes(graph: Graph, language: str = "en") -> list[dict]:
     query = """
     SELECT ?class ?name ?description ?parentClass ?reference ?match ?comment
     WHERE {
-        #Finding IMF - elements
+        # Finding IMF - elements
         VALUES ?type { imf:BlockType imf:TerminalType imf:AttributeType }
         ?imfClass a ?type .
         OPTIONAL {?imfClass rdfs:subClassOf ?parent }.
         OPTIONAL {?imfClass rdfs:label | skos:prefLabel ?name }.
-        OPTIONAL {?imfClass rdfs:comment | skos:description ?description} .
+
+        # Note: Bug in PCA has lead to the use non-existing term skos:description. This will be replaced
+        # with the correct skos:definition in the near future, so both terms are included here.
+        OPTIONAL {?imfClass rdfs:comment | skos:definition | skos:description ?description} .
 
         # Finding the last segment of the class IRI
         BIND(STR(?imfClass) AS ?classString)
-        BIND(REPLACE(?classString, "^.*[/#]([^/#]*)$", "$1") AS ?classSegment)
+        BIND(REPLACE(?classString, "^.*[/#]([^/#]*)$", "$1") AS ?tempSegment)
+        BIND(REPLACE(?tempSegment, "-", "_") AS ?classSegment)
         BIND(IF(CONTAINS(?classString, "imf/"), CONCAT("IMF_", ?classSegment) , ?classSegment) AS ?class)
 
         # Add imf:Attribute as parent class

cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py

@@ -19,12 +19,12 @@ def parse_imf_metadata() -> dict:
     raw_metadata = {
         "role": RoleTypes.information,
         "schema": SchemaCompleteness.partial,
-        "prefix": "pca-imf",
-        "namespace": Namespace("http://posccaesar.org/imf/"),
+        "prefix": "pcaimf",
+        "namespace": Namespace("https://posccaesar.org/imf/"),
         "version": None,
         "created": None,
         "updated": None,
-        "title": "IMF - types",
+        "title": "IMF_types",
         "description": "IMF - types",
         "creator": None,
         "rights": None,

cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py

@@ -36,7 +36,7 @@ def parse_imf_to_properties(graph: Graph, language: str = "en") -> list[dict]:
     """
 
     query = """
-    SELECT ?class ?property ?name ?description ?valueType ?minCount ?maxCount ?default ?reference
+    SELECT DISTINCT ?class ?property ?name ?description ?valueType ?minCount ?maxCount ?default ?reference
     ?match ?comment ?propertyType
     WHERE
     {
@@ -48,7 +48,7 @@ def parse_imf_to_properties(graph: Graph, language: str = "en") -> list[dict]:
                 ?propertyShape sh:path ?imfProperty .
 
             OPTIONAL { ?imfProperty skos:prefLabel ?name . }
-            OPTIONAL { ?imfProperty skos:description ?description . }
+            OPTIONAL { ?imfProperty skos:definition | skos:description ?description . }
             OPTIONAL { ?imfProperty rdfs:range ?range . }
             OPTIONAL { ?imfProperty a ?type . }
             OPTIONAL { ?propertyShape sh:minCount ?minCardinality} .
@@ -62,22 +62,24 @@ def parse_imf_to_properties(graph: Graph, language: str = "en") -> list[dict]:
             ?imfClass a imf:AttributeType ;
                 ?imfProperty ?default .
 
-            # imf:predicate is required
-            BIND(IF(?imfProperty = <http://ns.imfid.org/imf#predicate>, 1, 0) AS ?minCardinality)
-
             # The following information is used to describe the attribute when it is connected to a block or a terminal
             # and not duplicated here.
-            FILTER(?imfProperty != rdf:type && ?imfProperty != skos:prefLabel && ?imfProperty != skos:description)
+            # Note: Bug in PCA has lead to the use non-existing term skos:description. This will be replaced
+            # with the correct skos:definition in the near future, so both terms are included here.
+            FILTER(?imfProperty != rdf:type && ?imfProperty != skos:prefLabel &&
+                ?imfProperty != skos:defintion && ?imfProperty != skos:description)
         }
 
         # Finding the last segment of the class IRI
         BIND(STR(?imfClass) AS ?classString)
-        BIND(REPLACE(?classString, "^.*[/#]([^/#]*)$", "$1") AS ?classSegment)
+        BIND(REPLACE(?classString, "^.*[/#]([^/#]*)$", "$1") AS ?tempClassSegment)
+        BIND(REPLACE(?tempClassSegment, "-", "_") AS ?classSegment)
         BIND(IF(CONTAINS(?classString, "imf/"), CONCAT("IMF_", ?classSegment) , ?classSegment) AS ?class)
 
         # Finding the last segment of the property IRI
         BIND(STR(?imfProperty) AS ?propertyString)
-        BIND(REPLACE(?propertyString, "^.*[/#]([^/#]*)$", "$1") AS ?propertySegment)
+        BIND(REPLACE(?propertyString, "^.*[/#]([^/#]*)$", "$1") AS ?tempPropertySegment)
+        BIND(REPLACE(?tempPropertySegment, "-", "_") AS ?propertySegment)
         BIND(IF(CONTAINS(?propertyString, "imf/"), CONCAT("IMF_", ?propertySegment) , ?propertySegment) AS ?property)
 
         # Set the value type for the property based on sh:class, sh:qualifiedValueType or rdfs:range
@@ -85,12 +87,17 @@ def parse_imf_to_properties(graph: Graph, language: str = "en") -> list[dict]:
 
         # Finding the last segment of value types
         BIND(STR(?valueIriType) AS ?valueTypeString)
-        BIND(REPLACE(?valueTypeString, "^.*[/#]([^/#]*)$", "$1") AS ?valueTypeSegment)
+        BIND(REPLACE(?valueTypeString, "^.*[/#]([^/#]*)$", "$1") AS ?tempValueTypeSegment)
+        BIND(REPLACE(?tempValueTypeSegment, "-", "_") AS ?valueTypeSegment)
         BIND(IF(CONTAINS(?valueTypeString, "imf/"), CONCAT("IMF_", ?valueTypeSegment) , ?valueTypeSegment)
             AS ?valueType)
 
-        # Helper variable to set property type if value type is not already set.
-        BIND(IF(BOUND(?type) && ?type = owl:DatatypeProperty, ?type , owl:ObjectProperty) AS ?propertyType)
+        # Helper variable to set owl datatype- or object-property if this is not already set.
+        BIND(IF( EXISTS {?imfProperty a ?tempPropertyType . FILTER(?tempPropertyType = owl:DatatypeProperty) },
+            owl:DatatypeProperty,
+            owl:ObjectProperty
+            )
+        AS ?propertyType)
 
         # Assert cardinality values if they do not exist
         BIND(IF(BOUND(?minCardinality), ?minCardinality, 0) AS ?minCount)

cognite/neat/rules/importers/_rdf/_imf2rules/_imf2rules.py

@@ -2,13 +2,14 @@
 there are loaders to TransformationRules pydantic class."""
 
 from pathlib import Path
-from typing import Literal, overload
 
 from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SH, SKOS, Graph
 
 from cognite.neat.issues import IssueList
-from cognite.neat.rules.importers._base import BaseImporter, VerifiedRules
-from cognite.neat.rules.models import InformationRules, RoleTypes
+from cognite.neat.issues.errors import FileReadError
+from cognite.neat.rules._shared import ReadRules
+from cognite.neat.rules.importers._base import BaseImporter
+from cognite.neat.rules.models import InformationInputRules
 from cognite.neat.rules.models.data_types import _XSD_TYPES
 
 from ._imf2classes import parse_imf_to_classes
@@ -16,7 +17,7 @@ from ._imf2metadata import parse_imf_metadata
 from ._imf2properties import parse_imf_to_properties
 
 
-class IMFImporter(BaseImporter):
+class IMFImporter(BaseImporter[InformationInputRules]):
     """Convert SHACL shapes to tables/ transformation rules / Excel file.
 
         Args:
@@ -38,24 +39,14 @@ class IMFImporter(BaseImporter):
     def __init__(self, filepath: Path):
         self.filepath = filepath
 
-    @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
-
-    @overload
-    def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
-    ) -> tuple[VerifiedRules | None, IssueList]: ...
-
     def to_rules(
         self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-    ) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
+    ) -> ReadRules[InformationInputRules]:
         graph = Graph()
         try:
             graph.parse(self.filepath)
         except Exception as e:
-            raise Exception(f"Could not parse owl file: {e}") from e
+            return ReadRules(None, IssueList([FileReadError(self.filepath, f"Could not parse owl file: {e}")]), {})
 
         # bind key namespaces
         graph.bind("owl", OWL)
@@ -76,8 +67,8 @@ class IMFImporter(BaseImporter):
 
         components = make_components_compliant(components)
 
-        rules = InformationRules.model_validate(components)
-        return self._to_output(rules, IssueList(), errors, role)
+        rules = InformationInputRules.load(components)
+        return ReadRules(rules, IssueList(), {})
 
 
 def make_components_compliant(components: dict) -> dict:

cognite/neat/rules/importers/_rdf/_inference2rules.py

@@ -1,20 +1,22 @@
 from collections import Counter, defaultdict
 from datetime import datetime
 from pathlib import Path
-from typing import Literal, cast, overload
+from typing import cast
 
-from rdflib import Graph, Namespace, URIRef
+from rdflib import RDF, Graph, Namespace, URIRef
 from rdflib import Literal as RdfLiteral
 
 from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
 from cognite.neat.issues import IssueList
 from cognite.neat.issues.errors import FileReadError
-from cognite.neat.rules.importers._base import BaseImporter, VerifiedRules, _handle_issues
-from cognite.neat.rules.models import InformationRules, RoleTypes
-from cognite.neat.rules.models._base import MatchType
+from cognite.neat.issues.warnings import PropertyValueTypeUndefinedWarning
+from cognite.neat.rules._shared import ReadRules
+from cognite.neat.rules.importers._base import BaseImporter
+from cognite.neat.rules.models._base_rules import MatchType
+from cognite.neat.rules.models.entities._single_value import UnknownEntity
 from cognite.neat.rules.models.information import (
+    InformationInputRules,
     InformationMetadata,
-    InformationRulesInput,
 )
 from cognite.neat.store import NeatGraphStore
 from cognite.neat.utils.rdf_ import get_namespace, remove_namespace_from_uri, uri_to_short_form
@@ -25,16 +27,6 @@ ORDERED_CLASSES_QUERY = """SELECT ?class (count(?s) as ?instances )
 
 INSTANCES_OF_CLASS_QUERY = """SELECT ?s WHERE { ?s a <class> . }"""
 
-INSTANCE_PROPERTIES_JSON_DEFINITION = """SELECT ?property (count(?property) as ?occurrence) ?dataType ?objectType
-                                    WHERE {<instance_id> ?property ?value .
-
-                                           BIND(IF(REGEX(?value, "^\u007b(.*)\u007d$"),
-                                           <http://www.w3.org/2001/XMLSchema#json>,
-                                           datatype(?value)) AS ?dataType)
-
-                                           OPTIONAL {?value rdf:type ?objectType .}}
-                                    GROUP BY ?property ?dataType ?objectType"""
-
 INSTANCE_PROPERTIES_DEFINITION = """SELECT ?property (count(?property) as ?occurrence) ?dataType ?objectType
                                     WHERE {<instance_id> ?property ?value .
 
@@ -44,7 +36,7 @@ INSTANCE_PROPERTIES_DEFINITION = """SELECT ?property (count(?property) as ?occur
                                     GROUP BY ?property ?dataType ?objectType"""
 
 
-class InferenceImporter(BaseImporter):
+class InferenceImporter(BaseImporter[InformationInputRules]):
     """Infers rules from a triple store.
 
     Rules inference through analysis of knowledge graph provided in various formats.
@@ -56,7 +48,6 @@ class InferenceImporter(BaseImporter):
         graph: Knowledge graph
         max_number_of_instance: Maximum number of instances to be used in inference
         prefix: Prefix to be used for the inferred model
-        check_for_json_string: Check if values are JSON strings
     """
 
     def __init__(
@@ -65,15 +56,11 @@ class InferenceImporter(BaseImporter):
         graph: Graph,
         max_number_of_instance: int = -1,
         prefix: str = "inferred",
-        check_for_json_string: bool = False,
     ) -> None:
         self.issue_list = issue_list
         self.graph = graph
         self.max_number_of_instance = max_number_of_instance
         self.prefix = prefix
-        self.check_for_json_string = (
-            check_for_json_string if graph.store.__class__.__name__ != "OxigraphStore" else False
-        )
 
     @classmethod
     def from_graph_store(
@@ -81,7 +68,6 @@ class InferenceImporter(BaseImporter):
         store: NeatGraphStore,
         max_number_of_instance: int = -1,
         prefix: str = "inferred",
-        check_for_json_string: bool = False,
     ) -> "InferenceImporter":
         issue_list = IssueList(title="Inferred from graph store")
 
@@ -90,7 +76,6 @@ class InferenceImporter(BaseImporter):
             store.graph,
             max_number_of_instance=max_number_of_instance,
             prefix=prefix,
-            check_for_json_string=check_for_json_string,
         )
 
     @classmethod
@@ -99,7 +84,6 @@ class InferenceImporter(BaseImporter):
         filepath: Path,
         max_number_of_instance: int = -1,
         prefix: str = "inferred",
-        check_for_json_string: bool = False,
     ) -> "InferenceImporter":
         issue_list = IssueList(title=f"'{filepath.name}'")
 
@@ -114,7 +98,6 @@ class InferenceImporter(BaseImporter):
             graph,
             max_number_of_instance=max_number_of_instance,
             prefix=prefix,
-            check_for_json_string=check_for_json_string,
         )
 
     @classmethod
@@ -123,7 +106,6 @@ class InferenceImporter(BaseImporter):
         filepath: Path,
         max_number_of_instance: int = -1,
         prefix: str = "inferred",
-        check_for_json_string: bool = False,
     ) -> "InferenceImporter":
         raise NotImplementedError("JSON file format is not supported yet.")
 
@@ -133,7 +115,6 @@ class InferenceImporter(BaseImporter):
         filepath: Path,
         max_number_of_instance: int = -1,
         prefix: str = "inferred",
-        check_for_json_string: bool = False,
     ) -> "InferenceImporter":
         raise NotImplementedError("YAML file format is not supported yet.")
 
@@ -143,48 +124,24 @@ class InferenceImporter(BaseImporter):
         filepath: Path,
         max_number_of_instance: int = -1,
         prefix: str = "inferred",
-        check_for_json_string: bool = False,
     ) -> "InferenceImporter":
         raise NotImplementedError("JSON file format is not supported yet.")
 
-    @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
-
-    @overload
-    def to_rules(
-        self,
-        errors: Literal["continue"] = "continue",
-        role: RoleTypes | None = None,
-    ) -> tuple[VerifiedRules | None, IssueList]: ...
-
     def to_rules(
         self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-    ) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
+    ) -> ReadRules[InformationInputRules]:
         """
         Creates `Rules` object from the data for target role.
         """
 
         if self.issue_list.has_errors:
             # In case there were errors during the import, the to_rules method will return None
-            return self._return_or_raise(self.issue_list, errors)
+            return ReadRules(None, self.issue_list, {})
 
         rules_dict = self._to_rules_components()
 
-        with _handle_issues(self.issue_list) as future:
-            rules: InformationRules
-            rules = InformationRulesInput.load(rules_dict).as_rules()
-
-        if future.result == "failure" or self.issue_list.has_errors:
-            return self._return_or_raise(self.issue_list, errors)
-
-        return self._to_output(
-            rules,
-            self.issue_list,
-            errors=errors,
-            role=role,
-        )
+        rules = InformationInputRules.load(rules_dict)
+        return ReadRules(rules, self.issue_list, {})
 
     def _to_rules_components(
         self,
@@ -203,7 +160,6 @@ class InferenceImporter(BaseImporter):
         prefixes: dict[str, Namespace] = get_default_prefixes()
         count_by_value_type_by_property: dict[str, dict[str, int]] = defaultdict(Counter)
 
-        query = INSTANCE_PROPERTIES_JSON_DEFINITION if self.check_for_json_string else INSTANCE_PROPERTIES_DEFINITION
         # Adds default namespace to prefixes
         prefixes[self._default_metadata().prefix] = self._default_metadata().namespace
 
@@ -231,19 +187,35 @@ class InferenceImporter(BaseImporter):
                 + f" LIMIT {self.max_number_of_instance}"
             ):
                 for property_uri, occurrence, data_type_uri, object_type_uri in self.graph.query(  # type: ignore[misc]
-                    query.replace("instance_id", instance)
+                    INSTANCE_PROPERTIES_DEFINITION.replace("instance_id", instance)
                 ):  # type: ignore[misc]
+                    # this is to skip rdf:type property
+                    if property_uri == RDF.type:
+                        continue
+
                     property_id = remove_namespace_from_uri(property_uri)
 
                     self._add_uri_namespace_to_prefixes(cast(URIRef, property_uri), prefixes)
-                    value_type_uri = data_type_uri if data_type_uri else object_type_uri
 
-                    # this is to skip rdf:type property
-                    if not value_type_uri:
-                        continue
+                    if value_type_uri := (data_type_uri or object_type_uri):
+                        self._add_uri_namespace_to_prefixes(cast(URIRef, value_type_uri), prefixes)
+
+                        value_type_id = remove_namespace_from_uri(value_type_uri)
+
+                    # this handles situations when property points to node that is not present in graph
+                    else:
+                        value_type_id = str(UnknownEntity())
+
+                        self.issue_list.append(
+                            PropertyValueTypeUndefinedWarning(
+                                resource_type="Property",
+                                identifier=f"{class_id}{property_id}",
+                                property_name=property_id,
+                                default_action="Remove the property from the rules",
+                                recommended_action="Make sure that graph is complete",
+                            )
+                        )
 
-                    self._add_uri_namespace_to_prefixes(cast(URIRef, value_type_uri), prefixes)
-                    value_type_id = remove_namespace_from_uri(value_type_uri)
                     id_ = f"{class_id}:{property_id}"
 
                     definition = {
@@ -288,7 +260,7 @@ class InferenceImporter(BaseImporter):
             if len(count_list) == 1:
                 type_, count = count_list[0]
                 counts_str = f"with value type {base_string.format(value_type=type_, count=count)} in the graph"
-            elif len(count_list) == 1:
+            elif len(count_list) == 2:
                 first = base_string.format(value_type=count_list[0][0], count=count_list[0][1])
                 second = base_string.format(value_type=count_list[1][0], count=count_list[1][1])
                 counts_str = f"with value types {first} and {second} in the graph"

cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py

@@ -2,21 +2,22 @@
 there are loaders to TransformationRules pydantic class."""
 
 from pathlib import Path
-from typing import Literal, overload
 
 from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SKOS, Graph
 
 from cognite.neat.issues import IssueList
-from cognite.neat.rules.importers._base import BaseImporter, VerifiedRules
+from cognite.neat.issues.errors import FileReadError
+from cognite.neat.rules._shared import ReadRules
+from cognite.neat.rules.importers._base import BaseImporter
 from cognite.neat.rules.importers._rdf._shared import make_components_compliant
-from cognite.neat.rules.models import InformationRules, RoleTypes
+from cognite.neat.rules.models import InformationInputRules
 
 from ._owl2classes import parse_owl_classes
 from ._owl2metadata import parse_owl_metadata
 from ._owl2properties import parse_owl_properties
 
 
-class OWLImporter(BaseImporter):
+class OWLImporter(BaseImporter[InformationInputRules]):
     """Convert OWL ontology to tables/ transformation rules / Excel file.
 
         Args:
@@ -37,24 +38,12 @@ class OWLImporter(BaseImporter):
     def __init__(self, filepath: Path):
         self.owl_filepath = filepath
 
-    @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
-
-    @overload
-    def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
-    ) -> tuple[VerifiedRules | None, IssueList]: ...
-
-    def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-    ) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
+    def to_rules(self) -> ReadRules[InformationInputRules]:
         graph = Graph()
         try:
             graph.parse(self.owl_filepath)
         except Exception as e:
-            raise Exception(f"Could not parse owl file: {e}") from e
+            return ReadRules(None, IssueList([FileReadError(self.owl_filepath, f"Could not parse owl file: {e}")]), {})
 
         # bind key namespaces
         graph.bind("owl", OWL)
@@ -72,5 +61,5 @@ class OWLImporter(BaseImporter):
 
         components = make_components_compliant(components)
 
-        rules = InformationRules.model_validate(components)
-        return self._to_output(rules, IssueList(), errors, role)
+        rules = InformationInputRules.load(components)
+        return ReadRules(rules, IssueList(), {})

cognite/neat/rules/importers/_rdf/_shared.py

@@ -4,7 +4,7 @@ import numpy as np
 import pandas as pd
 from rdflib import OWL, Literal, Namespace
 
-from cognite.neat.rules.models._base import MatchType
+from cognite.neat.rules.models._base_rules import MatchType
 from cognite.neat.rules.models.data_types import _XSD_TYPES
 from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 from cognite.neat.utils.regex_patterns import PATTERNS