cognite-neat 0.88.1__py3-none-any.whl → 0.88.3__py3-none-any.whl

cognite/neat/rules/importers/_owl2rules/_owl2metadata.py

@@ -1,209 +0,0 @@
-import datetime
-
-from rdflib import Graph, Namespace
-
-from cognite.neat.constants import DEFAULT_NAMESPACE
-from cognite.neat.rules.models import RoleTypes, SchemaCompleteness
-from cognite.neat.utils.collection_ import remove_none_elements_from_set
-from cognite.neat.utils.rdf_ import convert_rdflib_content
-from cognite.neat.utils.regex_patterns import PATTERNS
-
-
-def parse_owl_metadata(graph: Graph) -> dict:
-    """Parse owl metadata from graph to dict.
-
-    Args:
-        graph: Graph containing owl metadata
-
-    Returns:
-        Dictionary containing owl metadata
-
-    !!! note "Compliant OWL metadata"
-        This makes the method very opinionated, but results in a compliant metadata.
-
-
-    """
-    # TODO: Move dataframe to dict representation
-
-    query = f"""SELECT ?namespace ?prefix ?version ?created ?updated ?title ?description ?creator ?rights ?license
-    WHERE {{
-        ?namespace a owl:Ontology .
-        OPTIONAL {{?namespace owl:versionInfo ?version }}.
-        OPTIONAL {{?namespace dcterms:creator ?creator }}.
-        OPTIONAL {{?namespace <{DEFAULT_NAMESPACE.prefix}> ?prefix }}.
-        OPTIONAL {{?namespace dcterms:title|rdfs:label|skos:prefLabel ?title }}.
-        OPTIONAL {{?namespace dcterms:modified ?updated }}.
-        OPTIONAL {{?namespace dcterms:created ?created }}.
-        OPTIONAL {{?namespace dcterms:description ?description }}.
-        OPTIONAL {{?namespace dcterms:rights|dc:rights ?rights }}.
-
-        OPTIONAL {{?namespace dcterms:license|dc:license ?license }}.
-        FILTER (!isBlank(?namespace))
-        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
-        FILTER (!bound(?title) || LANG(?title) = "" || LANGMATCHES(LANG(?title), "en"))
-    }}
-    """
-
-    results = [{item for item in sublist} for sublist in list(zip(*graph.query(query), strict=True))]
-
-    raw_metadata = convert_rdflib_content(
-        {
-            "role": RoleTypes.information,
-            "schema": SchemaCompleteness.partial,
-            "prefix": results[1].pop(),
-            "namespace": Namespace(results[0].pop()),
-            "version": results[2].pop(),
-            "created": results[3].pop(),
-            "updated": results[4].pop(),
-            "title": results[5].pop(),
-            "description": results[6].pop(),
-            "creator": (
-                ", ".join(remove_none_elements_from_set(results[7]))
-                if remove_none_elements_from_set(results[7])
-                else None
-            ),
-            "rights": results[8].pop(),
-            "license": results[9].pop(),
-        }
-    )
-
-    return make_metadata_compliant(raw_metadata)
-
-
-def make_metadata_compliant(metadata: dict) -> dict:
-    """Attempts to fix errors in metadata, otherwise defaults to values that will pass validation.
-
-    Args:
-        metadata: Dictionary containing metadata
-
-    Returns:
-        Dictionary containing metadata with fixed errors
-    """
-
-    metadata = fix_namespace(metadata, default=Namespace("http://purl.org/cognite/neat#"))
-    metadata = fix_prefix(metadata)
-    metadata = fix_version(metadata)
-    metadata = fix_date(metadata, date_type="created", default=datetime.datetime.now().replace(microsecond=0))
-    metadata = fix_date(metadata, date_type="updated", default=datetime.datetime.now().replace(microsecond=0))
-    metadata = fix_title(metadata)
-    metadata = fix_description(metadata)
-    metadata = fix_author(metadata, "creator")
-    metadata = fix_rights(metadata)
-    metadata = fix_license(metadata)
-
-    return metadata
-
-
-def fix_license(metadata: dict, default: str = "Unknown license") -> dict:
-    if license := metadata.get("license", None):
-        if not isinstance(license, str):
-            metadata["license"] = default
-        elif isinstance(license, str) and len(license) == 0:
-            metadata["license"] = default
-    else:
-        metadata["license"] = default
-    return metadata
-
-
-def fix_rights(metadata: dict, default: str = "Unknown rights") -> dict:
-    if rights := metadata.get("rights", None):
-        if not isinstance(rights, str):
-            metadata["rights"] = default
-        elif isinstance(rights, str) and len(rights) == 0:
-            metadata["rights"] = default
-    else:
-        metadata["rights"] = default
-    return metadata
-
-
-def fix_author(metadata: dict, author_type: str = "creator", default: str = "NEAT") -> dict:
-    if author := metadata.get(author_type, None):
-        if not isinstance(author, str) or isinstance(author, list):
-            metadata[author_type] = default
-        elif isinstance(author, str) and len(author) == 0:
-            metadata[author_type] = default
-    else:
-        metadata[author_type] = default
-    return metadata
-
-
-def fix_description(metadata: dict, default: str = "This model has been inferred from OWL ontology") -> dict:
-    if description := metadata.get("description", None):
-        if not isinstance(description, str) or len(description) == 0:
-            metadata["description"] = default
-        elif isinstance(description, str) and len(description) > 1024:
-            metadata["description"] = metadata["description"][:1024]
-    else:
-        metadata["description"] = default
-    return metadata
-
-
-def fix_prefix(metadata: dict, default: str = "neat") -> dict:
-    if prefix := metadata.get("prefix", None):
-        if not isinstance(prefix, str) or not PATTERNS.prefix_compliance.match(prefix):
-            metadata["prefix"] = default
-    else:
-        metadata["prefix"] = default
-    return metadata
-
-
-def fix_namespace(metadata: dict, default: Namespace) -> dict:
-    if namespace := metadata.get("namespace", None):
-        if not isinstance(namespace, Namespace):
-            try:
-                metadata["namespace"] = Namespace(namespace)
-            except Exception:
-                metadata["namespace"] = default
-    else:
-        metadata["namespace"] = default
-
-    return metadata
-
-
-def fix_date(
-    metadata: dict,
-    date_type: str,
-    default: datetime.datetime,
-) -> dict:
-    if date := metadata.get(date_type, None):
-        try:
-            if isinstance(date, datetime.datetime):
-                return metadata
-            elif isinstance(date, datetime.date):
-                metadata[date_type] = datetime.datetime.combine(metadata[date_type], datetime.datetime.min.time())
-            elif isinstance(date, str):
-                metadata[date_type] = datetime.datetime.strptime(metadata[date_type], "%Y-%m-%dT%H:%M:%SZ")
-            else:
-                metadata[date_type] = default
-        except Exception:
-            metadata[date_type] = default
-    else:
-        metadata[date_type] = default
-
-    return metadata
-
-
-def fix_version(metadata: dict, default: str = "1.0.0") -> dict:
-    if version := metadata.get("version", None):
-        if not PATTERNS.version_compliance.match(version):
-            metadata["version"] = default
-    else:
-        metadata["version"] = default
-
-    return metadata
-
-
-def fix_title(metadata: dict, default: str = "OWL Inferred Data Model") -> dict:
-    if title := metadata.get("title", None):
-        if not isinstance(title, str):
-            metadata["title"] = default
-        elif isinstance(title, str) and len(title) == 0:
-            metadata["title"] = default
-        elif isinstance(title, str) and len(title) > 255:
-            metadata["title"] = metadata["title"][:255]
-        else:
-            pass
-    else:
-        metadata["title"] = default
-
-    return metadata

cognite/neat/rules/importers/_owl2rules/_owl2properties.py

@@ -1,203 +0,0 @@
-from typing import cast
-
-import numpy as np
-import pandas as pd
-from rdflib import Graph
-
-from cognite.neat.rules.models._base import MatchType
-from cognite.neat.utils.rdf_ import remove_namespace_from_uri
-
-from ._owl2classes import _data_type_property_class, _object_property_class, _thing_class
-
-
-def parse_owl_properties(graph: Graph, language: str = "en") -> list[dict]:
-    """Parse owl properties from graph to pandas dataframe.
-
-    Args:
-        graph: Graph containing owl properties
-        language: Language to use for parsing, by default "en"
-
-    Returns:
-        List of dictionaries containing owl properties
-    """
-
-    query = """
-
-    SELECT ?class ?property ?name ?description ?type ?minCount ?maxCount ?reference
-     ?match ?propertyType
-    WHERE {
-        ?property a ?propertyType.
-        FILTER (?propertyType IN (owl:ObjectProperty, owl:DatatypeProperty ) )
-        OPTIONAL {?property rdfs:domain ?class }.
-        OPTIONAL {?property rdfs:range ?type }.
-        OPTIONAL {?property rdfs:label ?name }.
-        OPTIONAL {?property rdfs:comment ?description} .
-        OPTIONAL {?property owl:maxCardinality ?maxCount} .
-        OPTIONAL {?property owl:minCardinality ?minCount} .
-        FILTER (!isBlank(?property))
-        FILTER (!bound(?type) || !isBlank(?type))
-        FILTER (!bound(?class) || !isBlank(?class))
-        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
-        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
-        BIND(IF(bound(?minCount), ?minCount, 0) AS ?minCount)
-        BIND(IF(bound(?maxCount), ?maxCount, 1) AS ?maxCount)
-    }
-    """
-
-    raw_df = _parse_raw_dataframe(cast(list[tuple], list(graph.query(query.replace("en", language)))))
-    if raw_df.empty:
-        return []
-
-    # group values and clean up
-    processed_df = _clean_up_properties(raw_df)
-
-    # make compliant
-    processed_df = make_properties_compliant(processed_df)
-
-    # drop column _property_type, which was a helper column:
-    processed_df.drop(columns=["_property_type"], inplace=True)
-
-    return processed_df.to_dict(orient="records")
-
-
-def _parse_raw_dataframe(query_results: list[tuple]) -> pd.DataFrame:
-    df = pd.DataFrame(
-        query_results,
-        columns=[
-            "Class",
-            "Property",
-            "Name",
-            "Description",
-            "Value Type",
-            "Min Count",
-            "Max Count",
-            "Reference",
-            "Match Type",
-            "_property_type",
-        ],
-    )
-    if df.empty:
-        return df
-
-    df.replace(np.nan, "", regex=True, inplace=True)
-
-    df.Reference = df.Property
-    df.Class = df.Class.apply(lambda x: remove_namespace_from_uri(x))
-    df.Property = df.Property.apply(lambda x: remove_namespace_from_uri(x))
-    df["Value Type"] = df["Value Type"].apply(lambda x: remove_namespace_from_uri(x))
-    df["Match Type"] = len(df) * [MatchType.exact]
-    df["Comment"] = len(df) * [None]
-    df["_property_type"] = df["_property_type"].apply(lambda x: remove_namespace_from_uri(x))
-
-    return df
-
-
-def _clean_up_properties(df: pd.DataFrame) -> pd.DataFrame:
-    class_grouped_dfs = df.groupby("Class")
-
-    clean_list = []
-
-    for class_, class_grouped_df in class_grouped_dfs:
-        property_grouped_dfs = class_grouped_df.groupby("Property")
-        for property_, property_grouped_df in property_grouped_dfs:
-            clean_list += [
-                {
-                    "Class": class_,
-                    "Property": property_,
-                    "Name": property_grouped_df["Name"].unique()[0],
-                    "Description": "\n".join(list(property_grouped_df.Description.unique()))[:1024],
-                    "Value Type": property_grouped_df["Value Type"].unique()[0],
-                    "Min Count": property_grouped_df["Min Count"].unique()[0],
-                    "Max Count": property_grouped_df["Max Count"].unique()[0],
-                    "Reference": property_grouped_df["Reference"].unique()[0],
-                    "Match Type": property_grouped_df["Match Type"].unique()[0],
-                    "Comment": property_grouped_df["Comment"].unique()[0],
-                    "_property_type": property_grouped_df["_property_type"].unique()[0],
-                }
-            ]
-
-    df = pd.DataFrame(clean_list)
-    df.replace("", None, inplace=True)
-
-    return df
-
-
-def make_properties_compliant(properties: pd.DataFrame) -> pd.DataFrame:
-    # default to None if "Min Count" is not specified
-    properties["Min Count"] = properties["Min Count"].apply(lambda x: 0 if not isinstance(x, int) or x == "" else x)
-
-    # default to None if "Max Count" is not specified
-    properties["Max Count"] = properties["Max Count"].apply(lambda x: 1 if not isinstance(x, int) or x == "" else x)
-
-    # Replace empty or non-string values in "Match Type" column with "exact"
-    properties["Match Type"] = properties["Match Type"].fillna("exact")
-    properties["Match Type"] = properties["Match Type"].apply(
-        lambda x: "exact" if not isinstance(x, str) or len(x) == 0 else x
-    )
-
-    # Replace empty or non-string values in "Comment" column with a default value
-    properties["Comment"] = properties["Comment"].fillna("Imported from Ontology by NEAT")
-    properties["Comment"] = properties["Comment"].apply(
-        lambda x: "Imported from Ontology by NEAT" if not isinstance(x, str) or len(x) == 0 else x
-    )
-
-    # Reduce length of elements in the "Description" column to 1024 characters
-    properties["Description"] = properties["Description"].apply(lambda x: x[:1024] if isinstance(x, str) else None)
-
-    # fixes and additions
-    properties = fix_dangling_properties(properties)
-    properties = fix_missing_property_value_type(properties)
-
-    return properties
-
-
-def fix_dangling_properties(properties: pd.DataFrame) -> pd.DataFrame:
-    """This method fixes properties which are missing a domain definition in the ontology.
-
-    Args:
-        properties: Dataframe containing properties
-
-    Returns:
-        Dataframe containing properties with fixed domain
-    """
-    domain = {
-        "ObjectProperty": _object_property_class()["Class"],
-        "DatatypeProperty": _data_type_property_class()["Class"],
-    }
-
-    # apply missing range
-    properties["Class"] = properties.apply(
-        lambda row: (
-            domain[row._property_type]
-            if row._property_type == "ObjectProperty" and pd.isna(row["Class"])
-            else domain["DatatypeProperty"]
-            if pd.isna(row["Class"])
-            else row["Class"]
-        ),
-        axis=1,
-    )
-    return properties
-
-
-def fix_missing_property_value_type(properties: pd.DataFrame) -> pd.DataFrame:
-    """This method fixes properties which are missing a range definition in the ontology.
-
-    Args:
-        properties: Dataframe containing properties
-
-    Returns:
-        Dataframe containing properties with fixed range
-    """
-    # apply missing range
-    properties["Value Type"] = properties.apply(
-        lambda row: (
-            _thing_class()["Class"]
-            if row._property_type == "ObjectProperty" and pd.isna(row["Value Type"])
-            else "string"
-            if pd.isna(row["Value Type"])
-            else row["Value Type"]
-        ),
-        axis=1,
-    )
-
-    return properties

cognite/neat/rules/issues/__init__.py

@@ -1,26 +0,0 @@
-from cognite.neat.issues import MultiValueError
-
-from . import dms, fileread, formatters, importing, spreadsheet, spreadsheet_file
-from .base import (
-    DefaultPydanticError,
-    IssueList,
-    NeatValidationError,
-    ValidationIssue,
-    ValidationWarning,
-)
-
-__all__ = [
-    "DefaultPydanticError",
-    "MultiValueError",
-    "IssueList",
-    "NeatValidationError",
-    "ValidationIssue",
-    "ValidationIssue",
-    "ValidationWarning",
-    "dms",
-    "fileread",
-    "formatters",
-    "importing",
-    "spreadsheet",
-    "spreadsheet_file",
-]

cognite/neat/rules/issues/base.py

@@ -1,82 +0,0 @@
-from abc import ABC
-from dataclasses import dataclass
-from typing import Any
-
-from pydantic_core import ErrorDetails
-
-from cognite.neat.issues import MultiValueError, NeatError, NeatIssue, NeatIssueList, NeatWarning
-
-__all__ = [
-    "ValidationIssue",
-    "NeatValidationError",
-    "DefaultPydanticError",
-    "ValidationWarning",
-    "IssueList",
-    "MultiValueError",
-]
-
-
-@dataclass(frozen=True)
-class ValidationIssue(NeatIssue, ABC): ...
-
-
-@dataclass(frozen=True)
-class NeatValidationError(NeatError, ValidationIssue, ABC):
-    @classmethod
-    def from_pydantic_errors(cls, errors: list[ErrorDetails], **kwargs) -> "list[NeatValidationError]":
-        """Convert a list of pydantic errors to a list of Error instances.
-
-        This is intended to be overridden in subclasses to handle specific error types.
-        """
-        all_errors: list[NeatValidationError] = []
-        for error in errors:
-            if isinstance(ctx := error.get("ctx"), dict) and isinstance(
-                multi_error := ctx.get("error"), MultiValueError
-            ):
-                all_errors.extend(multi_error.errors)  # type: ignore[arg-type]
-            else:
-                all_errors.append(DefaultPydanticError.from_pydantic_error(error))
-        return all_errors
-
-
-@dataclass(frozen=True)
-class DefaultPydanticError(NeatValidationError):
-    type: str
-    loc: tuple[int | str, ...]
-    msg: str
-    input: Any
-    ctx: dict[str, Any] | None
-
-    @classmethod
-    def from_pydantic_error(cls, error: ErrorDetails) -> "NeatValidationError":
-        return cls(
-            type=error["type"],
-            loc=error["loc"],
-            msg=error["msg"],
-            input=error.get("input"),
-            ctx=error.get("ctx"),
-        )
-
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["type"] = self.type
-        output["loc"] = self.loc
-        output["msg"] = self.msg
-        output["input"] = self.input
-        output["ctx"] = self.ctx
-        return output
-
-    def message(self) -> str:
-        if self.loc and len(self.loc) == 1:
-            return f"{self.loc[0]} sheet: {self.msg}"
-        elif self.loc and len(self.loc) == 2:
-            return f"{self.loc[0]} sheet field/column <{self.loc[1]}>: {self.msg}"
-        else:
-            return self.msg
-
-
-@dataclass(frozen=True)
-class ValidationWarning(NeatWarning, ValidationIssue, ABC): ...
-
-
-class IssueList(NeatIssueList[ValidationIssue]): ...