cognite-neat 0.88.0__py3-none-any.whl → 0.88.2__py3-none-any.whl

cognite/neat/rules/importers/_owl2rules/_owl2classes.py

@@ -1,215 +0,0 @@
-from typing import cast
-
-import numpy as np
-import pandas as pd
-from rdflib import OWL, Graph
-
-from cognite.neat.rules.models._base import MatchType
-from cognite.neat.utils.rdf_ import remove_namespace_from_uri
-
-
-def parse_owl_classes(graph: Graph, language: str = "en") -> list[dict]:
-    """Parse owl classes from graph to pandas dataframe.
-
-    Args:
-        graph: Graph containing owl classes
-        language: Language to use for parsing, by default "en"
-
-    Returns:
-        Dataframe containing owl classes
-
-    !!! note "Compliant OWL classes"
-        This makes the method very opinionated, but results in a compliant classes.
-    """
-
-    query = """
-        SELECT ?class ?name ?description ?parentClass ?reference ?match ?comment
-        WHERE {
-        ?class a owl:Class .
-        OPTIONAL {?class rdfs:subClassOf ?parentClass }.
-        OPTIONAL {?class rdfs:label ?name }.
-        OPTIONAL {?class rdfs:comment ?description} .
-        FILTER (!isBlank(?class))
-        FILTER (!bound(?parentClass) || !isBlank(?parentClass))
-        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
-        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
-    }
-    """
-
-    # create raw dataframe
-
-    raw_df = _parse_raw_dataframe(cast(list[tuple], list(graph.query(query.replace("en", language)))))
-    if raw_df.empty:
-        return []
-
-    # group values and clean up
-    processed_df = _clean_up_classes(raw_df)
-
-    # make compliant
-    processed_df = make_classes_compliant(processed_df)
-
-    # Make Parent Class list elements into string joined with comma
-    processed_df["Parent Class"] = processed_df["Parent Class"].apply(
-        lambda x: ", ".join(x) if isinstance(x, list) and x else None
-    )
-
-    return processed_df.dropna(axis=0, how="all").replace(float("nan"), None).to_dict(orient="records")
-
-
-def _parse_raw_dataframe(query_results: list[tuple]) -> pd.DataFrame:
-    df = pd.DataFrame(
-        query_results,
-        columns=["Class", "Name", "Description", "Parent Class", "Reference", "Match", "Comment"],
-    )
-    if df.empty:
-        return df
-
-    # # remove NaNs
-    df.replace(np.nan, "", regex=True, inplace=True)
-
-    df.Reference = df.Class
-    df.Class = df.Class.apply(lambda x: remove_namespace_from_uri(x))
-    df["Match Type"] = len(df) * [MatchType.exact]
-    df["Comment"] = len(df) * [None]
-    df["Parent Class"] = df["Parent Class"].apply(lambda x: remove_namespace_from_uri(x))
-
-    return df
-
-
-def _clean_up_classes(df: pd.DataFrame) -> pd.DataFrame:
-    clean_list = [
-        {
-            "Class": class_,
-            "Name": group_df["Name"].unique()[0],
-            "Description": "\n".join(list(group_df.Description.unique())),
-            "Parent Class": ", ".join(list(group_df["Parent Class"].unique())),
-            "Reference": group_df["Reference"].unique()[0],
-            "Match Type": group_df["Match Type"].unique()[0],
-            "Comment": group_df["Comment"].unique()[0],
-        }
-        for class_, group_df in df.groupby("Class")
-    ]
-
-    df = pd.DataFrame(clean_list)
-
-    # bring NaNs back
-    df.replace("", None, inplace=True)
-
-    # split Parent Class column back into list
-    df["Parent Class"] = df["Parent Class"].apply(lambda x: x.split(", ") if isinstance(x, str) else None)
-
-    return df
-
-
-def make_classes_compliant(classes: pd.DataFrame) -> pd.DataFrame:
-    """Make classes compliant.
-
-    Returns:
-        Dataframe containing compliant classes
-
-    !!! note "About the compliant classes"
-        The compliant classes are based on the OWL base ontology, but adapted to NEAT and use in CDF.
-        One thing to note is that this method would not be able to fix issues with class ids which
-        are not compliant with the CDF naming convention. For example, if a class id contains a space,
-        starts with a number, etc. This will cause issues when trying to create the class in CDF.
-    """
-
-    # Replace empty or non-string values in "Match" column with "exact"
-    classes["Match Type"] = classes["Match Type"].fillna(MatchType.exact)
-    classes["Match Type"] = classes["Match Type"].apply(
-        lambda x: MatchType.exact if not isinstance(x, str) or len(x) == 0 else x
-    )
-
-    # Replace empty or non-string values in "Comment" column with a default value
-    classes["Comment"] = classes["Comment"].fillna("Imported from Ontology by NEAT")
-    classes["Comment"] = classes["Comment"].apply(
-        lambda x: "Imported from Ontology by NEAT" if not isinstance(x, str) or len(x) == 0 else x
-    )
-
-    # Add _object_property_class, _data_type_property_class, _thing_class to the dataframe
-    classes = pd.concat(
-        [classes, pd.DataFrame([_object_property_class(), _data_type_property_class(), _thing_class()])],
-        ignore_index=True,
-    )
-
-    # Reduce length of elements in the "Description" column to 1024 characters
-    classes["Description"] = classes["Description"].apply(lambda x: x[:1024] if isinstance(x, str) else None)
-
-    # Add missing parent classes to the dataframe
-    classes = pd.concat(
-        [classes, pd.DataFrame(_add_parent_class(classes))],
-        ignore_index=True,
-    )
-
-    return classes
-
-
-def _object_property_class() -> dict:
-    return {
-        "Class": "ObjectProperty",
-        "Name": None,
-        "Description": "The class of object properties.",
-        "Parent Class": None,
-        "Reference": OWL.ObjectProperty,
-        "Match Type": MatchType.exact,
-        "Comment": "Added by NEAT based on owl:ObjectProperty but adapted to NEAT and use in CDF.",
-    }
-
-
-def _data_type_property_class() -> dict:
-    return {
-        "Class": "DatatypeProperty",
-        "Name": None,
-        "Description": "The class of data properties.",
-        "Parent Class": None,
-        "Reference": OWL.DatatypeProperty,
-        "Match Type": MatchType.exact,
-        "Comment": "Added by NEAT based on owl:DatatypeProperty but adapted to NEAT and use in CDF.",
-    }
-
-
-def _thing_class() -> dict:
-    return {
-        "Class": "Thing",
-        "Name": None,
-        "Description": "The class of holding class individuals.",
-        "Parent Class": None,
-        "Reference": OWL.Thing,
-        "Match Type": MatchType.exact,
-        "Comment": (
-            "Added by NEAT. "
-            "Imported from OWL base ontology, it is meant for use as a default"
-            " value type for object properties which miss a declared range."
-        ),
-    }
-
-
-def _add_parent_class(df: pd.DataFrame) -> list[dict]:
-    parent_set = {
-        item
-        for sublist in df["Parent Class"].tolist()
-        if sublist
-        for item in sublist
-        if item != "" and item is not None
-    }
-    class_set = set(df["Class"].tolist())
-
-    rows = []
-    for missing_parent_class in parent_set.difference(class_set):
-        rows += [
-            {
-                "Class": missing_parent_class,
-                "Name": None,
-                "Description": None,
-                "Parent Class": None,
-                "Reference": None,
-                "Match Type": None,
-                "Comment": (
-                    "Added by NEAT. "
-                    "This is a parent class that is missing in the ontology. "
-                    "It is added by NEAT to make the ontology compliant with CDF."
-                ),
-            }
-        ]
-
-    return rows

cognite/neat/rules/importers/_owl2rules/_owl2metadata.py

@@ -1,213 +0,0 @@
-import datetime
-import re
-
-from rdflib import Graph, Namespace
-
-from cognite.neat.constants import DEFAULT_NAMESPACE
-from cognite.neat.rules.models import RoleTypes, SchemaCompleteness
-from cognite.neat.rules.models._types._base import (
-    PREFIX_COMPLIANCE_REGEX,
-    VERSION_COMPLIANCE_REGEX,
-)
-from cognite.neat.utils.collection_ import remove_none_elements_from_set
-from cognite.neat.utils.rdf_ import convert_rdflib_content
-
-
-def parse_owl_metadata(graph: Graph) -> dict:
-    """Parse owl metadata from graph to dict.
-
-    Args:
-        graph: Graph containing owl metadata
-
-    Returns:
-        Dictionary containing owl metadata
-
-    !!! note "Compliant OWL metadata"
-        This makes the method very opinionated, but results in a compliant metadata.
-
-
-    """
-    # TODO: Move dataframe to dict representation
-
-    query = f"""SELECT ?namespace ?prefix ?version ?created ?updated ?title ?description ?creator ?rights ?license
-    WHERE {{
-        ?namespace a owl:Ontology .
-        OPTIONAL {{?namespace owl:versionInfo ?version }}.
-        OPTIONAL {{?namespace dcterms:creator ?creator }}.
-        OPTIONAL {{?namespace <{DEFAULT_NAMESPACE.prefix}> ?prefix }}.
-        OPTIONAL {{?namespace dcterms:title|rdfs:label|skos:prefLabel ?title }}.
-        OPTIONAL {{?namespace dcterms:modified ?updated }}.
-        OPTIONAL {{?namespace dcterms:created ?created }}.
-        OPTIONAL {{?namespace dcterms:description ?description }}.
-        OPTIONAL {{?namespace dcterms:rights|dc:rights ?rights }}.
-
-        OPTIONAL {{?namespace dcterms:license|dc:license ?license }}.
-        FILTER (!isBlank(?namespace))
-        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
-        FILTER (!bound(?title) || LANG(?title) = "" || LANGMATCHES(LANG(?title), "en"))
-    }}
-    """
-
-    results = [{item for item in sublist} for sublist in list(zip(*graph.query(query), strict=True))]
-
-    raw_metadata = convert_rdflib_content(
-        {
-            "role": RoleTypes.information,
-            "schema": SchemaCompleteness.partial,
-            "prefix": results[1].pop(),
-            "namespace": Namespace(results[0].pop()),
-            "version": results[2].pop(),
-            "created": results[3].pop(),
-            "updated": results[4].pop(),
-            "title": results[5].pop(),
-            "description": results[6].pop(),
-            "creator": (
-                ", ".join(remove_none_elements_from_set(results[7]))
-                if remove_none_elements_from_set(results[7])
-                else None
-            ),
-            "rights": results[8].pop(),
-            "license": results[9].pop(),
-        }
-    )
-
-    return make_metadata_compliant(raw_metadata)
-
-
-def make_metadata_compliant(metadata: dict) -> dict:
-    """Attempts to fix errors in metadata, otherwise defaults to values that will pass validation.
-
-    Args:
-        metadata: Dictionary containing metadata
-
-    Returns:
-        Dictionary containing metadata with fixed errors
-    """
-
-    metadata = fix_namespace(metadata, default=Namespace("http://purl.org/cognite/neat#"))
-    metadata = fix_prefix(metadata)
-    metadata = fix_version(metadata)
-    metadata = fix_date(metadata, date_type="created", default=datetime.datetime.now().replace(microsecond=0))
-    metadata = fix_date(metadata, date_type="updated", default=datetime.datetime.now().replace(microsecond=0))
-    metadata = fix_title(metadata)
-    metadata = fix_description(metadata)
-    metadata = fix_author(metadata, "creator")
-    metadata = fix_rights(metadata)
-    metadata = fix_license(metadata)
-
-    return metadata
-
-
-def fix_license(metadata: dict, default: str = "Unknown license") -> dict:
-    if license := metadata.get("license", None):
-        if not isinstance(license, str):
-            metadata["license"] = default
-        elif isinstance(license, str) and len(license) == 0:
-            metadata["license"] = default
-    else:
-        metadata["license"] = default
-    return metadata
-
-
-def fix_rights(metadata: dict, default: str = "Unknown rights") -> dict:
-    if rights := metadata.get("rights", None):
-        if not isinstance(rights, str):
-            metadata["rights"] = default
-        elif isinstance(rights, str) and len(rights) == 0:
-            metadata["rights"] = default
-    else:
-        metadata["rights"] = default
-    return metadata
-
-
-def fix_author(metadata: dict, author_type: str = "creator", default: str = "NEAT") -> dict:
-    if author := metadata.get(author_type, None):
-        if not isinstance(author, str) or isinstance(author, list):
-            metadata[author_type] = default
-        elif isinstance(author, str) and len(author) == 0:
-            metadata[author_type] = default
-    else:
-        metadata[author_type] = default
-    return metadata
-
-
-def fix_description(metadata: dict, default: str = "This model has been inferred from OWL ontology") -> dict:
-    if description := metadata.get("description", None):
-        if not isinstance(description, str) or len(description) == 0:
-            metadata["description"] = default
-        elif isinstance(description, str) and len(description) > 1024:
-            metadata["description"] = metadata["description"][:1024]
-    else:
-        metadata["description"] = default
-    return metadata
-
-
-def fix_prefix(metadata: dict, default: str = "neat") -> dict:
-    if prefix := metadata.get("prefix", None):
-        if not isinstance(prefix, str) or not re.match(PREFIX_COMPLIANCE_REGEX, prefix):
-            metadata["prefix"] = default
-    else:
-        metadata["prefix"] = default
-    return metadata
-
-
-def fix_namespace(metadata: dict, default: Namespace) -> dict:
-    if namespace := metadata.get("namespace", None):
-        if not isinstance(namespace, Namespace):
-            try:
-                metadata["namespace"] = Namespace(namespace)
-            except Exception:
-                metadata["namespace"] = default
-    else:
-        metadata["namespace"] = default
-
-    return metadata
-
-
-def fix_date(
-    metadata: dict,
-    date_type: str,
-    default: datetime.datetime,
-) -> dict:
-    if date := metadata.get(date_type, None):
-        try:
-            if isinstance(date, datetime.datetime):
-                return metadata
-            elif isinstance(date, datetime.date):
-                metadata[date_type] = datetime.datetime.combine(metadata[date_type], datetime.datetime.min.time())
-            elif isinstance(date, str):
-                metadata[date_type] = datetime.datetime.strptime(metadata[date_type], "%Y-%m-%dT%H:%M:%SZ")
-            else:
-                metadata[date_type] = default
-        except Exception:
-            metadata[date_type] = default
-    else:
-        metadata[date_type] = default
-
-    return metadata
-
-
-def fix_version(metadata: dict, default: str = "1.0.0") -> dict:
-    if version := metadata.get("version", None):
-        if not re.match(VERSION_COMPLIANCE_REGEX, version):
-            metadata["version"] = default
-    else:
-        metadata["version"] = default
-
-    return metadata
-
-
-def fix_title(metadata: dict, default: str = "OWL Inferred Data Model") -> dict:
-    if title := metadata.get("title", None):
-        if not isinstance(title, str):
-            metadata["title"] = default
-        elif isinstance(title, str) and len(title) == 0:
-            metadata["title"] = default
-        elif isinstance(title, str) and len(title) > 255:
-            metadata["title"] = metadata["title"][:255]
-        else:
-            pass
-    else:
-        metadata["title"] = default
-
-    return metadata

cognite/neat/rules/importers/_owl2rules/_owl2properties.py

@@ -1,203 +0,0 @@
-from typing import cast
-
-import numpy as np
-import pandas as pd
-from rdflib import Graph
-
-from cognite.neat.rules.models._base import MatchType
-from cognite.neat.utils.rdf_ import remove_namespace_from_uri
-
-from ._owl2classes import _data_type_property_class, _object_property_class, _thing_class
-
-
-def parse_owl_properties(graph: Graph, language: str = "en") -> list[dict]:
-    """Parse owl properties from graph to pandas dataframe.
-
-    Args:
-        graph: Graph containing owl properties
-        language: Language to use for parsing, by default "en"
-
-    Returns:
-        List of dictionaries containing owl properties
-    """
-
-    query = """
-
-    SELECT ?class ?property ?name ?description ?type ?minCount ?maxCount ?reference
-     ?match ?propertyType
-    WHERE {
-        ?property a ?propertyType.
-        FILTER (?propertyType IN (owl:ObjectProperty, owl:DatatypeProperty ) )
-        OPTIONAL {?property rdfs:domain ?class }.
-        OPTIONAL {?property rdfs:range ?type }.
-        OPTIONAL {?property rdfs:label ?name }.
-        OPTIONAL {?property rdfs:comment ?description} .
-        OPTIONAL {?property owl:maxCardinality ?maxCount} .
-        OPTIONAL {?property owl:minCardinality ?minCount} .
-        FILTER (!isBlank(?property))
-        FILTER (!bound(?type) || !isBlank(?type))
-        FILTER (!bound(?class) || !isBlank(?class))
-        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
-        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
-        BIND(IF(bound(?minCount), ?minCount, 0) AS ?minCount)
-        BIND(IF(bound(?maxCount), ?maxCount, 1) AS ?maxCount)
-    }
-    """
-
-    raw_df = _parse_raw_dataframe(cast(list[tuple], list(graph.query(query.replace("en", language)))))
-    if raw_df.empty:
-        return []
-
-    # group values and clean up
-    processed_df = _clean_up_properties(raw_df)
-
-    # make compliant
-    processed_df = make_properties_compliant(processed_df)
-
-    # drop column _property_type, which was a helper column:
-    processed_df.drop(columns=["_property_type"], inplace=True)
-
-    return processed_df.to_dict(orient="records")
-
-
-def _parse_raw_dataframe(query_results: list[tuple]) -> pd.DataFrame:
-    df = pd.DataFrame(
-        query_results,
-        columns=[
-            "Class",
-            "Property",
-            "Name",
-            "Description",
-            "Value Type",
-            "Min Count",
-            "Max Count",
-            "Reference",
-            "Match Type",
-            "_property_type",
-        ],
-    )
-    if df.empty:
-        return df
-
-    df.replace(np.nan, "", regex=True, inplace=True)
-
-    df.Reference = df.Property
-    df.Class = df.Class.apply(lambda x: remove_namespace_from_uri(x))
-    df.Property = df.Property.apply(lambda x: remove_namespace_from_uri(x))
-    df["Value Type"] = df["Value Type"].apply(lambda x: remove_namespace_from_uri(x))
-    df["Match Type"] = len(df) * [MatchType.exact]
-    df["Comment"] = len(df) * [None]
-    df["_property_type"] = df["_property_type"].apply(lambda x: remove_namespace_from_uri(x))
-
-    return df
-
-
-def _clean_up_properties(df: pd.DataFrame) -> pd.DataFrame:
-    class_grouped_dfs = df.groupby("Class")
-
-    clean_list = []
-
-    for class_, class_grouped_df in class_grouped_dfs:
-        property_grouped_dfs = class_grouped_df.groupby("Property")
-        for property_, property_grouped_df in property_grouped_dfs:
-            clean_list += [
-                {
-                    "Class": class_,
-                    "Property": property_,
-                    "Name": property_grouped_df["Name"].unique()[0],
-                    "Description": "\n".join(list(property_grouped_df.Description.unique()))[:1024],
-                    "Value Type": property_grouped_df["Value Type"].unique()[0],
-                    "Min Count": property_grouped_df["Min Count"].unique()[0],
-                    "Max Count": property_grouped_df["Max Count"].unique()[0],
-                    "Reference": property_grouped_df["Reference"].unique()[0],
-                    "Match Type": property_grouped_df["Match Type"].unique()[0],
-                    "Comment": property_grouped_df["Comment"].unique()[0],
-                    "_property_type": property_grouped_df["_property_type"].unique()[0],
-                }
-            ]
-
-    df = pd.DataFrame(clean_list)
-    df.replace("", None, inplace=True)
-
-    return df
-
-
-def make_properties_compliant(properties: pd.DataFrame) -> pd.DataFrame:
-    # default to None if "Min Count" is not specified
-    properties["Min Count"] = properties["Min Count"].apply(lambda x: 0 if not isinstance(x, int) or x == "" else x)
-
-    # default to None if "Max Count" is not specified
-    properties["Max Count"] = properties["Max Count"].apply(lambda x: 1 if not isinstance(x, int) or x == "" else x)
-
-    # Replace empty or non-string values in "Match Type" column with "exact"
-    properties["Match Type"] = properties["Match Type"].fillna("exact")
-    properties["Match Type"] = properties["Match Type"].apply(
-        lambda x: "exact" if not isinstance(x, str) or len(x) == 0 else x
-    )
-
-    # Replace empty or non-string values in "Comment" column with a default value
-    properties["Comment"] = properties["Comment"].fillna("Imported from Ontology by NEAT")
-    properties["Comment"] = properties["Comment"].apply(
-        lambda x: "Imported from Ontology by NEAT" if not isinstance(x, str) or len(x) == 0 else x
-    )
-
-    # Reduce length of elements in the "Description" column to 1024 characters
-    properties["Description"] = properties["Description"].apply(lambda x: x[:1024] if isinstance(x, str) else None)
-
-    # fixes and additions
-    properties = fix_dangling_properties(properties)
-    properties = fix_missing_property_value_type(properties)
-
-    return properties
-
-
-def fix_dangling_properties(properties: pd.DataFrame) -> pd.DataFrame:
-    """This method fixes properties which are missing a domain definition in the ontology.
-
-    Args:
-        properties: Dataframe containing properties
-
-    Returns:
-        Dataframe containing properties with fixed domain
-    """
-    domain = {
-        "ObjectProperty": _object_property_class()["Class"],
-        "DatatypeProperty": _data_type_property_class()["Class"],
-    }
-
-    # apply missing range
-    properties["Class"] = properties.apply(
-        lambda row: (
-            domain[row._property_type]
-            if row._property_type == "ObjectProperty" and pd.isna(row["Class"])
-            else domain["DatatypeProperty"]
-            if pd.isna(row["Class"])
-            else row["Class"]
-        ),
-        axis=1,
-    )
-    return properties
-
-
-def fix_missing_property_value_type(properties: pd.DataFrame) -> pd.DataFrame:
-    """This method fixes properties which are missing a range definition in the ontology.
-
-    Args:
-        properties: Dataframe containing properties
-
-    Returns:
-        Dataframe containing properties with fixed range
-    """
-    # apply missing range
-    properties["Value Type"] = properties.apply(
-        lambda row: (
-            _thing_class()["Class"]
-            if row._property_type == "ObjectProperty" and pd.isna(row["Value Type"])
-            else "string"
-            if pd.isna(row["Value Type"])
-            else row["Value Type"]
-        ),
-        axis=1,
-    )
-
-    return properties