cognite-neat 0.98.0__py3-none-any.whl → 0.99.1__py3-none-any.whl

cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py

@@ -1,29 +0,0 @@
-from cognite.neat._rules.importers._rdf._shared import make_metadata_compliant
-from cognite.neat._rules.models import RoleTypes
-
-
-def parse_imf_metadata(space: str = "pcaimf") -> dict:
-    """Provide hardcoded IMF metadata to dict.
-
-    Returns:
-        Dictionary containing IMF metadata
-
-    !!! note "Compliant IMF metadata"
-        The current RDF provide IMF types as SHACL, but there are not any metadata describing
-        the actual content.
-
-    """
-
-    raw_metadata = {
-        "role": RoleTypes.information,
-        "space": space,
-        "external_id": "imf_types",
-        "version": None,
-        "created": None,
-        "updated": None,
-        "name": "IMF Types",
-        "description": "IMF - types",
-        "creator": None,
-    }
-
-    return make_metadata_compliant(raw_metadata)

cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py

@@ -1,130 +0,0 @@
-from typing import cast
-
-from rdflib import Graph
-
-from cognite.neat._rules.importers._rdf._shared import (
-    clean_up_properties,
-    make_properties_compliant,
-    parse_raw_properties_dataframe,
-)
-
-
-def parse_imf_to_properties(graph: Graph, language: str = "en") -> list[dict]:
-    """Parse IMF elements from RDF-graph and extract properties to pandas dataframe.
-
-    Args:
-        graph: Graph containing imf elements
-        language: Language to use for parsing, by default "en"
-
-    Returns:
-        List of dictionaries containing properties extracted from IMF elements
-
-    !!! note "IMF Compliance"
-        The IMF elements are expressed in RDF, primarily using SHACL and OWL. To ensure
-        that the resulting properties are compliant with CDF, similar validation checks
-        as in the OWL ontology importer are applied.
-
-        For the IMF-types more of the compliance logic is placed directly in the SPARQL
-        query. Among these are the creation of class and property names not starting
-        with a number, ensuring property types as well as default cardinality boundraries.
-
-        IMF-attributes are considered both classes and properties. This kind of punning
-        is necessary to capture additional information carried by attributes. They carry,
-        among other things, a set of relationsships to reference terms, units of measure,
-        and qualifiers that together make up the meaning of the attribute. These references
-        are listed as additional properties with default values.
-    """
-
-    query = """
-    SELECT DISTINCT ?class ?property ?name ?description ?valueType ?minCount ?maxCount ?default ?propertyType
-    WHERE
-    {
-        # Finding IMF-blocks and terminals
-        {
-            VALUES ?classType { imf:BlockType imf:TerminalType }
-            ?imfClass a ?classType ;
-                sh:property ?propertyShape .
-                ?propertyShape sh:path ?imfProperty .
-
-            OPTIONAL { ?imfProperty skos:prefLabel ?name . }
-            OPTIONAL { ?imfProperty skos:definition | skos:description ?description . }
-            OPTIONAL { ?imfProperty rdfs:range ?range . }
-            OPTIONAL { ?imfProperty a ?type . }
-            OPTIONAL { ?propertyShape sh:minCount ?minCardinality} .
-            OPTIONAL { ?propertyShape sh:maxCount ?maxCardinality} .
-            OPTIONAL { ?propertyShape sh:hasValue ?defualt . }
-            OPTIONAL { ?propertyShape sh:class | sh:qualifiedValueShape/sh:class ?valueShape .}
-        }
-        UNION
-        # Finding the IMF-attribute types
-        {
-            ?imfClass a imf:AttributeType ;
-                ?imfProperty ?default .
-
-            # The following information is used to describe the attribute when it is connected to a block or a terminal
-            # and not duplicated here.
-            # Note: Bug in PCA has lead to the use non-existing term skos:description. This will be replaced
-            # with the correct skos:definition in the near future, so both terms are included here.
-            FILTER(?imfProperty != rdf:type && ?imfProperty != skos:prefLabel &&
-                ?imfProperty != skos:defintion && ?imfProperty != skos:description)
-        }
-
-        # Finding the last segment of the class IRI
-        BIND(STR(?imfClass) AS ?classString)
-        BIND(REPLACE(?classString, "^.*[/#]([^/#]*)$", "$1") AS ?tempClassSegment)
-        BIND(REPLACE(?tempClassSegment, "-", "_") AS ?classSegment)
-        BIND(IF(CONTAINS(?classString, "imf/"), CONCAT("IMF_", ?classSegment) , ?classSegment) AS ?class)
-
-
-        # Finding the last segment of the property IRI
-        BIND(STR(?imfProperty) AS ?propertyString)
-        BIND(REPLACE(?propertyString, "^.*[/#]([^/#]*)$", "$1") AS ?tempPropertySegment)
-        BIND(REPLACE(?tempPropertySegment, "-", "_") AS ?propertySegment)
-        BIND(IF(CONTAINS(?propertyString, "imf/"), CONCAT("IMF_", ?propertySegment) , ?propertySegment) AS ?property)
-
-        # Set the value type for the property based on sh:class, sh:qualifiedValueType or rdfs:range
-        BIND(IF(BOUND(?valueShape), ?valueShape, IF(BOUND(?range) , ?range , ?valueShape)) AS ?valueIriType)
-
-        # Finding the last segment of value types
-        BIND(STR(?valueIriType) AS ?valueTypeString)
-        BIND(REPLACE(?valueTypeString, "^.*[/#]([^/#]*)$", "$1") AS ?tempValueTypeSegment)
-        BIND(REPLACE(?tempValueTypeSegment, "-", "_") AS ?valueTypeSegment)
-        BIND(IF(CONTAINS(?valueTypeString, "imf/"), CONCAT("IMF_", ?valueTypeSegment) , ?valueTypeSegment)
-            AS ?valueType)
-
-        # Helper variable to set owl datatype- or object-property if this is not already set.
-        BIND(IF( EXISTS {?imfProperty a ?tempPropertyType . FILTER(?tempPropertyType = owl:DatatypeProperty) },
-            owl:DatatypeProperty,
-            owl:ObjectProperty
-            )
-        AS ?propertyType)
-
-        # Assert cardinality values if they do not exist
-        BIND(IF(BOUND(?minCardinality), ?minCardinality, 0) AS ?minCount)
-        BIND(IF(BOUND(?maxCardinality), ?maxCardinality, 1) AS ?maxCount)
-
-        # Rebind the IRI of the IMF-attribute to the ?reference variable to align with dataframe column headers
-        # This is solely for readability, the ?imfClass could have been returnered directly instead of ?reference
-        BIND(?imfProperty AS ?reference)
-
-        FILTER (!isBlank(?property))
-        FILTER (!bound(?class) || !isBlank(?class))
-        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
-        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
-    }
-    """
-
-    raw_df = parse_raw_properties_dataframe(cast(list[tuple], list(graph.query(query.replace("en", language)))))
-    if raw_df.empty:
-        return []
-
-    # group values and clean up
-    processed_df = clean_up_properties(raw_df)
-
-    # make compliant
-    processed_df = make_properties_compliant(processed_df, importer="IMF")
-
-    # drop column _property_type, which was a helper column:
-    processed_df.drop(columns=["_property_type"], inplace=True)
-
-    return processed_df.to_dict(orient="records")

cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2rules.py

@@ -1,154 +0,0 @@
-"""This module performs importing of various formats to one of serializations for which
-there are loaders to TransformationRules pydantic class."""
-
-from cognite.neat._rules.importers._rdf._base import BaseRDFImporter
-from cognite.neat._rules.models.data_types import _XSD_TYPES
-
-from ._imf2classes import parse_imf_to_classes
-from ._imf2metadata import parse_imf_metadata
-from ._imf2properties import parse_imf_to_properties
-
-
-class IMFImporter(BaseRDFImporter):
-    """Convert SHACL shapes to tables/ transformation rules / Excel file.
-
-        Args:
-            filepath: Path to RDF file containing the SHACL Shapes
-
-    !!! Note
-        Rewrite to fit the SHACL rules we apply
-        OWL Ontologies are information models which completeness varies. As such, constructing functional
-        data model directly will often be impossible, therefore the produced Rules object will be ill formed.
-        To avoid this, neat will automatically attempt to make the imported rules compliant by adding default
-        values for missing information, attaching dangling properties to default containers based on the
-        property type, etc.
-
-        One has to be aware that NEAT will be opinionated about how to make the ontology
-        compliant, and that the resulting rules may not be what you expect.
-
-    """
-
-    def _to_rules_components(
-        self,
-    ) -> dict:
-        components = {
-            "Metadata": parse_imf_metadata(),
-            "Classes": parse_imf_to_classes(self.graph),
-            "Properties": parse_imf_to_properties(self.graph),
-        }
-
-        return make_components_compliant(components)
-
-
-def make_components_compliant(components: dict) -> dict:
-    components = _add_missing_classes(components)
-    components = _add_missing_value_types(components)
-    components = _add_default_property_to_dangling_classes(components)
-
-    return components
-
-
-def _add_missing_classes(components: dict[str, list[dict]]) -> dict:
-    """Add missing classes to Classes.
-
-    Args:
-        tables: imported tables from owl ontology
-
-    Returns:
-        Updated tables with missing classes added to containers
-    """
-
-    missing_classes = {definition["Class"] for definition in components["Properties"]} - {
-        definition["Class"] for definition in components["Classes"]
-    }
-
-    comment = (
-        "Added by NEAT. "
-        "This is a class that a domain of a property but was not defined in the ontology. "
-        "It is added by NEAT to make the ontology compliant with CDF."
-    )
-
-    for class_ in missing_classes:
-        components["Classes"].append({"Class": class_, "Comment": comment})
-
-    return components
-
-
-def _add_missing_value_types(components: dict) -> dict:
-    """Add properties to classes that do not have any properties defined to them
-
-    Args:
-        tables: imported tables from owl ontology
-
-    Returns:
-        Updated tables with missing properties added to containers
-    """
-
-    xsd_types = _XSD_TYPES
-    candidate_value_types = {definition["Value Type"] for definition in components["Properties"]} - {
-        definition["Class"] for definition in components["Classes"]
-    }
-
-    # to avoid issue of case sensitivity for xsd types
-    value_types_lower = {v.lower() for v in candidate_value_types}
-
-    xsd_types_lower = {x.lower() for x in xsd_types}
-
-    # Create a mapping from lowercase strings to original strings
-    value_types_mapping = {v.lower(): v for v in candidate_value_types}
-
-    # Find the difference
-    difference = value_types_lower - xsd_types_lower
-
-    # Convert the difference back to the original case
-    difference_original_case = {value_types_mapping[d] for d in difference}
-
-    for class_ in difference_original_case:
-        components["Classes"].append(
-            {
-                "Class": class_,
-                "Comment": (
-                    "Added by NEAT. "
-                    "This is a class that a domain of a property but was not defined in the ontology. "
-                    "It is added by NEAT to make the ontology compliant with CDF."
-                ),
-            }
-        )
-
-    return components
-
-
-def _add_default_property_to_dangling_classes(components: dict[str, list[dict]]) -> dict:
-    """Add missing classes to Classes.
-
-    Args:
-        tables: imported tables from owl ontology
-
-    Returns:
-        Updated tables with missing classes added to containers
-    """
-
-    dangling_classes = {
-        definition["Class"] for definition in components["Classes"] if not definition.get("Parent Class", None)
-    } - {definition["Class"] for definition in components["Properties"]}
-
-    comment = (
-        "Added by NEAT. "
-        "This is property has been added to this class since otherwise it will create "
-        "dangling classes in the ontology."
-    )
-
-    for class_ in dangling_classes:
-        components["Properties"].append(
-            {
-                "Class": class_,
-                "Property": "label",
-                "Value Type": "string",
-                "Comment": comment,
-                "Min Count": 0,
-                "Max Count": 1,
-                "Reference": "http://www.w3.org/2000/01/rdf-schema#label",
-            }
-        )
-
-    return components

cognite/neat/_rules/importers/_rdf/_owl2rules/__init__.py

@@ -1,3 +0,0 @@
-from ._owl2rules import OWLImporter
-
-__all__ = ["OWLImporter"]

cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py

@@ -1,58 +0,0 @@
-from typing import cast
-
-from rdflib import Graph
-
-from cognite.neat._rules.importers._rdf._shared import (
-    clean_up_classes,
-    make_classes_compliant,
-    parse_raw_classes_dataframe,
-)
-
-
-def parse_owl_classes(graph: Graph, language: str = "en") -> list[dict]:
-    """Parse owl classes from graph to pandas dataframe.
-
-    Args:
-        graph: Graph containing owl classes
-        language: Language to use for parsing, by default "en"
-
-    Returns:
-        Dataframe containing owl classes
-
-    !!! note "Compliant OWL classes"
-        This makes the method very opinionated, but results in a compliant classes.
-    """
-
-    query = """
-        SELECT ?class ?name ?description ?parentClass
-        WHERE {
-        ?class a owl:Class .
-        OPTIONAL {?class rdfs:subClassOf ?parentClass }.
-        OPTIONAL {?class rdfs:label ?name }.
-        OPTIONAL {?class rdfs:comment ?description} .
-        FILTER (!isBlank(?class))
-        FILTER (!bound(?parentClass) || !isBlank(?parentClass))
-        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
-        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
-        BIND(?class AS ?reference)
-    }
-    """
-
-    # create raw dataframe
-
-    raw_df = parse_raw_classes_dataframe(cast(list[tuple], list(graph.query(query.replace("en", language)))))
-    if raw_df.empty:
-        return []
-
-    # group values and clean up
-    processed_df = clean_up_classes(raw_df)
-
-    # make compliant
-    processed_df = make_classes_compliant(processed_df, importer="OWL")
-
-    # Make Parent Class list elements into string joined with comma
-    processed_df["Implements"] = processed_df["Implements"].apply(
-        lambda x: ", ".join(x) if isinstance(x, list) and x else None
-    )
-
-    return processed_df.dropna(axis=0, how="all").replace(float("nan"), None).to_dict(orient="records")

cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py

@@ -1,65 +0,0 @@
-from rdflib import Graph
-
-from cognite.neat._constants import DEFAULT_NAMESPACE
-from cognite.neat._rules.importers._rdf._shared import make_metadata_compliant
-from cognite.neat._rules.models import RoleTypes
-from cognite.neat._utils.collection_ import remove_none_elements_from_set
-from cognite.neat._utils.rdf_ import convert_rdflib_content
-
-
-def parse_owl_metadata(graph: Graph) -> dict:
-    """Parse owl metadata from graph to dict.
-
-    Args:
-        graph: Graph containing owl metadata
-
-    Returns:
-        Dictionary containing owl metadata
-
-    !!! note "Compliant OWL metadata"
-        This makes the method very opinionated, but results in a compliant metadata.
-
-
-    """
-    # TODO: Move dataframe to dict representation
-
-    query = f"""SELECT ?namespace ?prefix ?version ?created ?updated ?title ?description ?creator ?rights ?license
-    WHERE {{
-        ?namespace a owl:Ontology .
-        OPTIONAL {{?namespace owl:versionInfo ?version }}.
-        OPTIONAL {{?namespace dcterms:creator ?creator }}.
-        OPTIONAL {{?namespace <{DEFAULT_NAMESPACE.prefix}> ?prefix }}.
-        OPTIONAL {{?namespace dcterms:title|rdfs:label|skos:prefLabel ?title }}.
-        OPTIONAL {{?namespace dcterms:modified ?updated }}.
-        OPTIONAL {{?namespace dcterms:created ?created }}.
-        OPTIONAL {{?namespace dcterms:description ?description }}.
-        OPTIONAL {{?namespace dcterms:rights|dc:rights ?rights }}.
-
-        OPTIONAL {{?namespace dcterms:license|dc:license ?license }}.
-        FILTER (!isBlank(?namespace))
-        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
-        FILTER (!bound(?title) || LANG(?title) = "" || LANGMATCHES(LANG(?title), "en"))
-    }}
-    """
-
-    results = [{item for item in sublist} for sublist in list(zip(*graph.query(query), strict=True))]
-
-    raw_metadata = convert_rdflib_content(
-        {
-            "role": RoleTypes.information,
-            "space": results[1].pop(),
-            "external_id": "OntologyBasedDataModel",
-            "version": results[2].pop(),
-            "created": results[3].pop(),
-            "updated": results[4].pop(),
-            "name": results[5].pop(),
-            "description": results[6].pop(),
-            "creator": (
-                ", ".join(remove_none_elements_from_set(results[7]))
-                if remove_none_elements_from_set(results[7])
-                else None
-            ),
-        }
-    )
-
-    return make_metadata_compliant(raw_metadata)

cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py

@@ -1,59 +0,0 @@
-from typing import cast
-
-from rdflib import Graph
-
-from cognite.neat._rules.importers._rdf._shared import (
-    clean_up_properties,
-    make_properties_compliant,
-    parse_raw_properties_dataframe,
-)
-
-
-def parse_owl_properties(graph: Graph, language: str = "en") -> list[dict]:
-    """Parse owl properties from graph to pandas dataframe.
-
-    Args:
-        graph: Graph containing owl properties
-        language: Language to use for parsing, by default "en"
-
-    Returns:
-        List of dictionaries containing owl properties
-    """
-
-    query = """
-
-    SELECT ?class ?property ?name ?description ?type ?minCount ?maxCount ?default ?propertyType
-    WHERE {
-        ?property a ?propertyType.
-        FILTER (?propertyType IN (owl:ObjectProperty, owl:DatatypeProperty ) )
-        OPTIONAL {?property rdfs:domain ?class }.
-        OPTIONAL {?property rdfs:range ?type }.
-        OPTIONAL {?property rdfs:label ?name }.
-        OPTIONAL {?property rdfs:comment ?description} .
-        OPTIONAL {?property owl:maxCardinality ?maxCount} .
-        OPTIONAL {?property owl:minCardinality ?minCount} .
-        FILTER (!isBlank(?property))
-        FILTER (!bound(?type) || !isBlank(?type))
-        FILTER (!bound(?class) || !isBlank(?class))
-        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
-        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
-        BIND(IF(bound(?minCount), ?minCount, 0) AS ?minCount)
-        BIND(IF(bound(?maxCount), ?maxCount, 1) AS ?maxCount)
-        BIND(?property AS ?reference)
-    }
-    """
-
-    raw_df = parse_raw_properties_dataframe(cast(list[tuple], list(graph.query(query.replace("en", language)))))
-    if raw_df.empty:
-        return []
-
-    # group values and clean up
-    processed_df = clean_up_properties(raw_df)
-
-    # make compliant
-    processed_df = make_properties_compliant(processed_df, importer="OWL")
-
-    # drop column _property_type, which was a helper column:
-    processed_df.drop(columns=["_property_type"], inplace=True)
-
-    return processed_df.to_dict(orient="records")

cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2rules.py

@@ -1,39 +0,0 @@
-"""This module performs importing of various formats to one of serializations for which
-there are loaders to TransformationRules pydantic class."""
-
-from cognite.neat._rules.importers._rdf._base import BaseRDFImporter
-from cognite.neat._rules.importers._rdf._shared import make_components_compliant
-
-from ._owl2classes import parse_owl_classes
-from ._owl2metadata import parse_owl_metadata
-from ._owl2properties import parse_owl_properties
-
-
-class OWLImporter(BaseRDFImporter):
-    """Convert OWL ontology to tables/ transformation rules / Excel file.
-
-        Args:
-            filepath: Path to OWL ontology
-
-    !!! Note
-        OWL Ontologies are information models which completeness varies. As such, constructing functional
-        data model directly will often be impossible, therefore the produced Rules object will be ill formed.
-        To avoid this, neat will automatically attempt to make the imported rules compliant by adding default
-        values for missing information, attaching dangling properties to default containers based on the
-        property type, etc.
-
-        One has to be aware that NEAT will be opinionated about how to make the ontology
-        compliant, and that the resulting rules may not be what you expect.
-
-    """
-
-    def _to_rules_components(
-        self,
-    ) -> dict:
-        components = {
-            "Metadata": parse_owl_metadata(self.graph),
-            "Classes": parse_owl_classes(self.graph),
-            "Properties": parse_owl_properties(self.graph),
-        }
-
-        return make_components_compliant(components)