cognite-neat 0.88.1__py3-none-any.whl → 0.88.2__py3-none-any.whl

cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py

@@ -0,0 +1,82 @@
+from typing import cast
+
+from rdflib import Graph
+
+from cognite.neat.rules.importers._rdf._shared import (
+    clean_up_classes,
+    make_classes_compliant,
+    parse_raw_classes_dataframe,
+)
+
+
+def parse_imf_to_classes(graph: Graph, language: str = "en") -> list[dict]:
+    """Parse IMF elements from RDF-graph and extract classes to pandas dataframe.
+
+    Args:
+        graph: Graph containing imf elements
+        language: Language to use for parsing, by default "en"
+
+    Returns:
+        Dataframe containing imf elements
+
+    !!! note "IMF Compliance"
+        The IMF elements are expressed in RDF, primarily using SHACL and OWL. To ensure
+        that the resulting classes are compliant with CDF, similar validation checks as
+        in the OWL ontology importer are applied.
+
+        For the IMF-types more of the compliance logic is placed directly in the SPARQL
+        query. Among these are the creation of class name not starting with a number,
+        and ensuring that all classes have a parent.
+
+        IMF-attributes are considered both classes and properties. This kind of punning
+        is necessary to capture additional information carried by attributes. They carry,
+        among other things, a set of relationsships to reference terms, units of measure,
+        and qualifiers that together make up the meaning of the attribute.
+    """
+
+    query = """
+    SELECT ?class ?name ?description ?parentClass ?reference ?match ?comment
+    WHERE {
+        #Finding IMF - elements
+        VALUES ?type { imf:BlockType imf:TerminalType imf:AttributeType }
+        ?imfClass a ?type .
+        OPTIONAL {?imfClass rdfs:subClassOf ?parent }.
+        OPTIONAL {?imfClass rdfs:label | skos:prefLabel ?name }.
+        OPTIONAL {?imfClass rdfs:comment | skos:description ?description} .
+
+        # Finding the last segment of the class IRI
+        BIND(STR(?imfClass) AS ?classString)
+        BIND(REPLACE(?classString, "^.*[/#]([^/#]*)$", "$1") AS ?classSegment)
+        BIND(IF(CONTAINS(?classString, "imf/"), CONCAT("IMF_", ?classSegment) , ?classSegment) AS ?class)
+
+        # Add imf:Attribute as parent class
+        BIND(IF(!bound(?parent) && ?type = imf:AttributeType, imf:Attribute, ?parent) AS ?parentClass)
+
+        # Rebind the IRI of the IMF-type to the ?reference variable to align with dataframe column headers
+        # This is solely for readability, the ?imfClass could have been returnered directly instead of ?reference
+        BIND(?imfClass AS ?reference)
+
+        FILTER (!isBlank(?class))
+        FILTER (!bound(?parentClass) || !isBlank(?parentClass))
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
+    }
+    """
+
+    # create raw dataframe
+    raw_df = parse_raw_classes_dataframe(cast(list[tuple], list(graph.query(query.replace("en", language)))))
+    if raw_df.empty:
+        return []
+
+    # group values and clean up
+    processed_df = clean_up_classes(raw_df)
+
+    # make compliant
+    processed_df = make_classes_compliant(processed_df, importer="IMF")
+
+    # Make Parent Class list elements into string joined with comma
+    processed_df["Parent Class"] = processed_df["Parent Class"].apply(
+        lambda x: ", ".join(x) if isinstance(x, list) and x else None
+    )
+
+    return processed_df.dropna(axis=0, how="all").replace(float("nan"), None).to_dict(orient="records")

cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py

@@ -0,0 +1,34 @@
+from rdflib import Namespace
+
+from cognite.neat.rules.importers._rdf._shared import make_metadata_compliant
+from cognite.neat.rules.models import RoleTypes, SchemaCompleteness
+
+
+def parse_imf_metadata() -> dict:
+    """Provide hardcoded IMF metadata to dict.
+
+    Returns:
+        Dictionary containing IMF metadata
+
+    !!! note "Compliant IMF metadata"
+        The current RDF provide IMF types as SHACL, but there are not any metadata describing
+        the actual content.
+
+    """
+
+    raw_metadata = {
+        "role": RoleTypes.information,
+        "schema": SchemaCompleteness.partial,
+        "prefix": "pca-imf",
+        "namespace": Namespace("http://posccaesar.org/imf/"),
+        "version": None,
+        "created": None,
+        "updated": None,
+        "title": "IMF - types",
+        "description": "IMF - types",
+        "creator": None,
+        "rights": None,
+        "license": None,
+    }
+
+    return make_metadata_compliant(raw_metadata)

cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py

@@ -0,0 +1,123 @@
+from typing import cast
+
+from rdflib import Graph
+
+from cognite.neat.rules.importers._rdf._shared import (
+    clean_up_properties,
+    make_properties_compliant,
+    parse_raw_properties_dataframe,
+)
+
+
+def parse_imf_to_properties(graph: Graph, language: str = "en") -> list[dict]:
+    """Parse IMF elements from RDF-graph and extract properties to pandas dataframe.
+
+    Args:
+        graph: Graph containing imf elements
+        language: Language to use for parsing, by default "en"
+
+    Returns:
+        List of dictionaries containing properties extracted from IMF elements
+
+    !!! note "IMF Compliance"
+        The IMF elements are expressed in RDF, primarily using SHACL and OWL. To ensure
+        that the resulting properties are compliant with CDF, similar validation checks
+        as in the OWL ontology importer are applied.
+
+        For the IMF-types more of the compliance logic is placed directly in the SPARQL
+        query. Among these are the creation of class and property names not starting
+        with a number, ensuring property types as well as default cardinality boundraries.
+
+        IMF-attributes are considered both classes and properties. This kind of punning
+        is necessary to capture additional information carried by attributes. They carry,
+        among other things, a set of relationsships to reference terms, units of measure,
+        and qualifiers that together make up the meaning of the attribute. These references
+        are listed as additional properties with default values.
+    """
+
+    query = """
+    SELECT ?class ?property ?name ?description ?valueType ?minCount ?maxCount ?default ?reference
+    ?match ?comment ?propertyType
+    WHERE
+    {
+        # Finding IMF-blocks and terminals
+        {
+            VALUES ?classType { imf:BlockType imf:TerminalType }
+            ?imfClass a ?classType ;
+                sh:property ?propertyShape .
+                ?propertyShape sh:path ?imfProperty .
+
+            OPTIONAL { ?imfProperty skos:prefLabel ?name . }
+            OPTIONAL { ?imfProperty skos:description ?description . }
+            OPTIONAL { ?imfProperty rdfs:range ?range . }
+            OPTIONAL { ?imfProperty a ?type . }
+            OPTIONAL { ?propertyShape sh:minCount ?minCardinality} .
+            OPTIONAL { ?propertyShape sh:maxCount ?maxCardinality} .
+            OPTIONAL { ?propertyShape sh:hasValue ?defualt . }
+            OPTIONAL { ?propertyShape sh:class | sh:qualifiedValueShape/sh:class ?valueShape .}
+        }
+        UNION
+        # Finding the IMF-attribute types
+        {
+            ?imfClass a imf:AttributeType ;
+                ?imfProperty ?default .
+
+            # imf:predicate is required
+            BIND(IF(?imfProperty = <http://ns.imfid.org/imf#predicate>, 1, 0) AS ?minCardinality)
+
+            # The following information is used to describe the attribute when it is connected to a block or a terminal
+            # and not duplicated here.
+            FILTER(?imfProperty != rdf:type && ?imfProperty != skos:prefLabel && ?imfProperty != skos:description)
+        }
+
+        # Finding the last segment of the class IRI
+        BIND(STR(?imfClass) AS ?classString)
+        BIND(REPLACE(?classString, "^.*[/#]([^/#]*)$", "$1") AS ?classSegment)
+        BIND(IF(CONTAINS(?classString, "imf/"), CONCAT("IMF_", ?classSegment) , ?classSegment) AS ?class)
+
+        # Finding the last segment of the property IRI
+        BIND(STR(?imfProperty) AS ?propertyString)
+        BIND(REPLACE(?propertyString, "^.*[/#]([^/#]*)$", "$1") AS ?propertySegment)
+        BIND(IF(CONTAINS(?propertyString, "imf/"), CONCAT("IMF_", ?propertySegment) , ?propertySegment) AS ?property)
+
+        # Set the value type for the property based on sh:class, sh:qualifiedValueType or rdfs:range
+        BIND(IF(BOUND(?valueShape), ?valueShape, IF(BOUND(?range) , ?range , ?valueShape)) AS ?valueIriType)
+
+        # Finding the last segment of value types
+        BIND(STR(?valueIriType) AS ?valueTypeString)
+        BIND(REPLACE(?valueTypeString, "^.*[/#]([^/#]*)$", "$1") AS ?valueTypeSegment)
+        BIND(IF(CONTAINS(?valueTypeString, "imf/"), CONCAT("IMF_", ?valueTypeSegment) , ?valueTypeSegment)
+            AS ?valueType)
+
+        # Helper variable to set property type if value type is not already set.
+        BIND(IF(BOUND(?type) && ?type = owl:DatatypeProperty, ?type , owl:ObjectProperty) AS ?propertyType)
+
+        # Assert cardinality values if they do not exist
+        BIND(IF(BOUND(?minCardinality), ?minCardinality, 0) AS ?minCount)
+        BIND(IF(BOUND(?maxCardinality), ?maxCardinality, 1) AS ?maxCount)
+
+        # Rebind the IRI of the IMF-attribute to the ?reference variable to align with dataframe column headers
+        # This is solely for readability, the ?imfClass could have been returnered directly instead of ?reference
+        BIND(?imfProperty AS ?reference)
+
+        FILTER (!isBlank(?property))
+        FILTER (!bound(?class) || !isBlank(?class))
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
+    }
+    """
+
+    raw_df = parse_raw_properties_dataframe(cast(list[tuple], list(graph.query(query.replace("en", language)))))
+    if raw_df.empty:
+        return []
+
+    # group values and clean up
+    processed_df = clean_up_properties(raw_df)
+
+    # make compliant
+    processed_df = make_properties_compliant(processed_df, importer="IMF")
+
+    # drop column _property_type, which was a helper column:
+    processed_df.drop(columns=["_property_type"], inplace=True)
+
+    return processed_df.to_dict(orient="records")

cognite/neat/rules/importers/{_owl2rules/_owl2rules.py → _rdf/_imf2rules/_imf2rules.py}

@@ -4,25 +4,26 @@ there are loaders to TransformationRules pydantic class."""
 from pathlib import Path
 from typing import Literal, overload
 
-from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SKOS, Graph
+from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SH, SKOS, Graph
 
+from cognite.neat.issues import IssueList
 from cognite.neat.rules.importers._base import BaseImporter, Rules
-from cognite.neat.rules.issues import IssueList
 from cognite.neat.rules.models import InformationRules, RoleTypes
 from cognite.neat.rules.models.data_types import _XSD_TYPES
 
-from ._owl2classes import parse_owl_classes
-from ._owl2metadata import parse_owl_metadata
-from ._owl2properties import parse_owl_properties
+from ._imf2classes import parse_imf_to_classes
+from ._imf2metadata import parse_imf_metadata
+from ._imf2properties import parse_imf_to_properties
 
 
-class OWLImporter(BaseImporter):
-    """Convert OWL ontology to tables/ transformation rules / Excel file.
+class IMFImporter(BaseImporter):
+    """Convert SHACL shapes to tables/ transformation rules / Excel file.
 
         Args:
-            filepath: Path to OWL ontology
+            filepath: Path to RDF file containing the SHACL Shapes
 
     !!! Note
+        Rewrite to fit the SHACL rules we apply
         OWL Ontologies are information models which completeness varies. As such, constructing functional
         data model directly will often be impossible, therefore the produced Rules object will be ill formed.
         To avoid this, neat will automatically attempt to make the imported rules compliant by adding default
@@ -61,11 +62,14 @@ class OWLImporter(BaseImporter):
         graph.bind("dcterms", DCTERMS)
         graph.bind("dc", DC)
         graph.bind("skos", SKOS)
+        graph.bind("sh", SH)
+        graph.bind("xsd", _XSD_TYPES)
+        graph.bind("imf", "http://ns.imfid.org/imf#")
 
         components = {
-            "Metadata": parse_owl_metadata(graph),
-            "Classes": parse_owl_classes(graph),
-            "Properties": parse_owl_properties(graph),
+            "Metadata": parse_imf_metadata(),
+            "Classes": parse_imf_to_classes(graph),
+            "Properties": parse_imf_to_properties(graph),
         }
 
         components = make_components_compliant(components)

cognite/neat/rules/importers/{_inference2rules.py → _rdf/_inference2rules.py}

@@ -9,8 +9,8 @@ from rdflib import Literal as RdfLiteral
 import cognite.neat.rules.issues as issues
 from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
 from cognite.neat.graph.stores import NeatGraphStore
+from cognite.neat.issues import IssueList
 from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
-from cognite.neat.rules.issues import IssueList
 from cognite.neat.rules.models import InformationRules, RoleTypes
 from cognite.neat.rules.models._base import MatchType
 from cognite.neat.rules.models.information import (

cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py

@@ -0,0 +1,57 @@
+from typing import cast
+
+from rdflib import Graph
+
+from cognite.neat.rules.importers._rdf._shared import (
+    clean_up_classes,
+    make_classes_compliant,
+    parse_raw_classes_dataframe,
+)
+
+
+def parse_owl_classes(graph: Graph, language: str = "en") -> list[dict]:
+    """Parse owl classes from graph to pandas dataframe.
+
+    Args:
+        graph: Graph containing owl classes
+        language: Language to use for parsing, by default "en"
+
+    Returns:
+        Dataframe containing owl classes
+
+    !!! note "Compliant OWL classes"
+        This makes the method very opinionated, but results in a compliant classes.
+    """
+
+    query = """
+        SELECT ?class ?name ?description ?parentClass ?reference ?match ?comment
+        WHERE {
+        ?class a owl:Class .
+        OPTIONAL {?class rdfs:subClassOf ?parentClass }.
+        OPTIONAL {?class rdfs:label ?name }.
+        OPTIONAL {?class rdfs:comment ?description} .
+        FILTER (!isBlank(?class))
+        FILTER (!bound(?parentClass) || !isBlank(?parentClass))
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
+    }
+    """
+
+    # create raw dataframe
+
+    raw_df = parse_raw_classes_dataframe(cast(list[tuple], list(graph.query(query.replace("en", language)))))
+    if raw_df.empty:
+        return []
+
+    # group values and clean up
+    processed_df = clean_up_classes(raw_df)
+
+    # make compliant
+    processed_df = make_classes_compliant(processed_df, importer="OWL")
+
+    # Make Parent Class list elements into string joined with comma
+    processed_df["Parent Class"] = processed_df["Parent Class"].apply(
+        lambda x: ", ".join(x) if isinstance(x, list) and x else None
+    )
+
+    return processed_df.dropna(axis=0, how="all").replace(float("nan"), None).to_dict(orient="records")

cognite/neat/rules/importers/_rdf/_owl2rules/_owl2metadata.py

@@ -0,0 +1,68 @@
+from rdflib import Graph, Namespace
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.rules.importers._rdf._shared import make_metadata_compliant
+from cognite.neat.rules.models import RoleTypes, SchemaCompleteness
+from cognite.neat.utils.collection_ import remove_none_elements_from_set
+from cognite.neat.utils.rdf_ import convert_rdflib_content
+
+
+def parse_owl_metadata(graph: Graph) -> dict:
+    """Parse owl metadata from graph to dict.
+
+    Args:
+        graph: Graph containing owl metadata
+
+    Returns:
+        Dictionary containing owl metadata
+
+    !!! note "Compliant OWL metadata"
+        This makes the method very opinionated, but results in a compliant metadata.
+
+
+    """
+    # TODO: Move dataframe to dict representation
+
+    query = f"""SELECT ?namespace ?prefix ?version ?created ?updated ?title ?description ?creator ?rights ?license
+    WHERE {{
+        ?namespace a owl:Ontology .
+        OPTIONAL {{?namespace owl:versionInfo ?version }}.
+        OPTIONAL {{?namespace dcterms:creator ?creator }}.
+        OPTIONAL {{?namespace <{DEFAULT_NAMESPACE.prefix}> ?prefix }}.
+        OPTIONAL {{?namespace dcterms:title|rdfs:label|skos:prefLabel ?title }}.
+        OPTIONAL {{?namespace dcterms:modified ?updated }}.
+        OPTIONAL {{?namespace dcterms:created ?created }}.
+        OPTIONAL {{?namespace dcterms:description ?description }}.
+        OPTIONAL {{?namespace dcterms:rights|dc:rights ?rights }}.
+
+        OPTIONAL {{?namespace dcterms:license|dc:license ?license }}.
+        FILTER (!isBlank(?namespace))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
+        FILTER (!bound(?title) || LANG(?title) = "" || LANGMATCHES(LANG(?title), "en"))
+    }}
+    """
+
+    results = [{item for item in sublist} for sublist in list(zip(*graph.query(query), strict=True))]
+
+    raw_metadata = convert_rdflib_content(
+        {
+            "role": RoleTypes.information,
+            "schema": SchemaCompleteness.partial,
+            "prefix": results[1].pop(),
+            "namespace": Namespace(results[0].pop()),
+            "version": results[2].pop(),
+            "created": results[3].pop(),
+            "updated": results[4].pop(),
+            "title": results[5].pop(),
+            "description": results[6].pop(),
+            "creator": (
+                ", ".join(remove_none_elements_from_set(results[7]))
+                if remove_none_elements_from_set(results[7])
+                else None
+            ),
+            "rights": results[8].pop(),
+            "license": results[9].pop(),
+        }
+    )
+
+    return make_metadata_compliant(raw_metadata)

cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py

@@ -0,0 +1,59 @@
+from typing import cast
+
+from rdflib import Graph
+
+from cognite.neat.rules.importers._rdf._shared import (
+    clean_up_properties,
+    make_properties_compliant,
+    parse_raw_properties_dataframe,
+)
+
+
+def parse_owl_properties(graph: Graph, language: str = "en") -> list[dict]:
+    """Parse owl properties from graph to pandas dataframe.
+
+    Args:
+        graph: Graph containing owl properties
+        language: Language to use for parsing, by default "en"
+
+    Returns:
+        List of dictionaries containing owl properties
+    """
+
+    query = """
+
+    SELECT ?class ?property ?name ?description ?type ?minCount ?maxCount ?default ?reference
+     ?match ?comment ?propertyType
+    WHERE {
+        ?property a ?propertyType.
+        FILTER (?propertyType IN (owl:ObjectProperty, owl:DatatypeProperty ) )
+        OPTIONAL {?property rdfs:domain ?class }.
+        OPTIONAL {?property rdfs:range ?type }.
+        OPTIONAL {?property rdfs:label ?name }.
+        OPTIONAL {?property rdfs:comment ?description} .
+        OPTIONAL {?property owl:maxCardinality ?maxCount} .
+        OPTIONAL {?property owl:minCardinality ?minCount} .
+        FILTER (!isBlank(?property))
+        FILTER (!bound(?type) || !isBlank(?type))
+        FILTER (!bound(?class) || !isBlank(?class))
+        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
+        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
+        BIND(IF(bound(?minCount), ?minCount, 0) AS ?minCount)
+        BIND(IF(bound(?maxCount), ?maxCount, 1) AS ?maxCount)
+    }
+    """
+
+    raw_df = parse_raw_properties_dataframe(cast(list[tuple], list(graph.query(query.replace("en", language)))))
+    if raw_df.empty:
+        return []
+
+    # group values and clean up
+    processed_df = clean_up_properties(raw_df)
+
+    # make compliant
+    processed_df = make_properties_compliant(processed_df, importer="OWL")
+
+    # drop column _property_type, which was a helper column:
+    processed_df.drop(columns=["_property_type"], inplace=True)
+
+    return processed_df.to_dict(orient="records")

cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py

@@ -0,0 +1,76 @@
+"""This module performs importing of various formats to one of serializations for which
+there are loaders to TransformationRules pydantic class."""
+
+from pathlib import Path
+from typing import Literal, overload
+
+from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SKOS, Graph
+
+from cognite.neat.issues import IssueList
+from cognite.neat.rules.importers._base import BaseImporter, Rules
+from cognite.neat.rules.importers._rdf._shared import make_components_compliant
+from cognite.neat.rules.models import InformationRules, RoleTypes
+
+from ._owl2classes import parse_owl_classes
+from ._owl2metadata import parse_owl_metadata
+from ._owl2properties import parse_owl_properties
+
+
+class OWLImporter(BaseImporter):
+    """Convert OWL ontology to tables/ transformation rules / Excel file.
+
+        Args:
+            filepath: Path to OWL ontology
+
+    !!! Note
+        OWL Ontologies are information models which completeness varies. As such, constructing functional
+        data model directly will often be impossible, therefore the produced Rules object will be ill formed.
+        To avoid this, neat will automatically attempt to make the imported rules compliant by adding default
+        values for missing information, attaching dangling properties to default containers based on the
+        property type, etc.
+
+        One has to be aware that NEAT will be opinionated about how to make the ontology
+        compliant, and that the resulting rules may not be what you expect.
+
+    """
+
+    def __init__(self, filepath: Path):
+        self.owl_filepath = filepath
+
+    @overload
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
+
+    @overload
+    def to_rules(
+        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
+    ) -> tuple[Rules | None, IssueList]: ...
+
+    def to_rules(
+        self,
+        errors: Literal["raise", "continue"] = "continue",
+        role: RoleTypes | None = None,
+    ) -> tuple[Rules | None, IssueList] | Rules:
+        graph = Graph()
+        try:
+            graph.parse(self.owl_filepath)
+        except Exception as e:
+            raise Exception(f"Could not parse owl file: {e}") from e
+
+        # bind key namespaces
+        graph.bind("owl", OWL)
+        graph.bind("rdf", RDF)
+        graph.bind("rdfs", RDFS)
+        graph.bind("dcterms", DCTERMS)
+        graph.bind("dc", DC)
+        graph.bind("skos", SKOS)
+
+        components = {
+            "Metadata": parse_owl_metadata(graph),
+            "Classes": parse_owl_classes(graph),
+            "Properties": parse_owl_properties(graph),
+        }
+
+        components = make_components_compliant(components)
+
+        rules = InformationRules.model_validate(components)
+        return self._to_output(rules, IssueList(), errors, role)