cognite-neat 0.87.6__py3-none-any.whl → 0.88.0__py3-none-any.whl

cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py

@@ -1,217 +0,0 @@
-from typing import cast
-
-import numpy as np
-import pandas as pd
-from rdflib import Graph
-
-from cognite.neat.utils.rdf_ import remove_namespace_from_uri
-
-from ._owl2classes import _data_type_property_class, _object_property_class, _thing_class
-
-
-def parse_owl_properties(graph: Graph, make_compliant: bool = False, language: str = "en") -> pd.DataFrame:
-    """Get all properties from the OWL ontology
-
-    Parameters
-    ----------
-    graph : Graph
-        Graph to query
-    parsing_config : dict, optional
-        Configuration for parsing the dataframe, by default None
-
-    Returns
-    -------
-    pd.DataFrame
-        Dataframe with columns: class, property, name, ...
-    """
-
-    query = """
-
-    SELECT ?class ?property ?name ?description ?type ?minCount ?maxCount
-     ?deprecated ?deprecationDate ?replacedBy ?source ?sourceEntity
-     ?match ?comment ?propertyType
-    WHERE {
-        ?property a ?propertyType.
-        FILTER (?propertyType IN (owl:ObjectProperty, owl:DatatypeProperty ) )
-        OPTIONAL {?property rdfs:domain ?class }.
-        OPTIONAL {?property rdfs:range ?type }.
-        OPTIONAL {?property rdfs:label ?name }.
-        OPTIONAL {?property rdfs:comment ?description} .
-        OPTIONAL {?property owl:maxCardinality ?maxCount} .
-        OPTIONAL {?property owl:minCardinality ?minCount} .
-        FILTER (!isBlank(?property))
-        FILTER (!bound(?type) || !isBlank(?type))
-        FILTER (!bound(?class) || !isBlank(?class))
-        FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
-        FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
-        OPTIONAL {?property owl:deprecated ?deprecated} .
-    }
-    """
-
-    raw_df = _parse_raw_dataframe(cast(list[tuple], list(graph.query(query.replace("en", language)))))
-    if raw_df.empty:
-        return pd.concat([raw_df, pd.DataFrame([len(raw_df) * [""]])], ignore_index=True)
-
-    # group values and clean up
-    processed_df = _clean_up_properties(raw_df)
-
-    # make compliant
-    if make_compliant:
-        processed_df = make_properties_compliant(processed_df)
-
-    # drop column _property_type, which was a helper column:
-    processed_df.drop(columns=["_property_type"], inplace=True)
-
-    return processed_df
-
-
-def _parse_raw_dataframe(query_results: list[tuple]) -> pd.DataFrame:
-    df = pd.DataFrame(
-        query_results,
-        columns=[
-            "Class",
-            "Property",
-            "Name",
-            "Description",
-            "Type",
-            "Min Count",
-            "Max Count",
-            "Deprecated",
-            "Deprecation Date",
-            "Replaced By",
-            "Source",
-            "Source Entity Name",
-            "Match Type",
-            "Comment",
-            "_property_type",
-        ],
-    )
-    if df.empty:
-        return df
-
-    df.replace(np.nan, "", regex=True, inplace=True)
-
-    df.Source = df.Property
-    df.Class = df.Class.apply(lambda x: remove_namespace_from_uri(x))
-    df.Property = df.Property.apply(lambda x: remove_namespace_from_uri(x))
-    df.Type = df.Type.apply(lambda x: remove_namespace_from_uri(x))
-    df["Source Entity Name"] = df.Property
-    df["Match Type"] = len(df) * ["exact"]
-    df["_property_type"] = df["_property_type"].apply(lambda x: remove_namespace_from_uri(x))
-
-    return df
-
-
-def _clean_up_properties(df: pd.DataFrame) -> pd.DataFrame:
-    class_grouped_dfs = df.groupby("Class")
-
-    clean_list = []
-
-    for class_, class_grouped_df in class_grouped_dfs:
-        property_grouped_dfs = class_grouped_df.groupby("Property")
-        for property_, property_grouped_df in property_grouped_dfs:
-            clean_list += [
-                {
-                    "Class": class_,
-                    "Property": property_,
-                    "Name": property_grouped_df["Name"].unique()[0],
-                    "Description": "\n".join(list(property_grouped_df.Description.unique()))[:1024],
-                    "Type": property_grouped_df.Type.unique()[0],
-                    "Min Count": property_grouped_df["Min Count"].unique()[0],
-                    "Max Count": property_grouped_df["Max Count"].unique()[0],
-                    "Deprecated": property_grouped_df.Deprecated.unique()[0],
-                    "Deprecation Date": property_grouped_df["Deprecation Date"].unique()[0],
-                    "Replaced By": property_grouped_df["Replaced By"].unique()[0],
-                    "Source": property_grouped_df["Source"].unique()[0],
-                    "Source Entity Name": property_grouped_df["Source Entity Name"].unique()[0],
-                    "Match Type": property_grouped_df["Match Type"].unique()[0],
-                    "Comment": property_grouped_df["Comment"].unique()[0],
-                    "_property_type": property_grouped_df["_property_type"].unique()[0],
-                }
-            ]
-
-    df = pd.DataFrame(clean_list)
-    df.replace("", None, inplace=True)
-
-    return df
-
-
-def make_properties_compliant(properties: pd.DataFrame) -> pd.DataFrame:
-    # default to None if "Min Count" is not specified
-    properties["Min Count"] = properties["Min Count"].apply(lambda x: None if not isinstance(x, int) or x == "" else x)
-
-    # default to None if "Max Count" is not specified
-    properties["Max Count"] = properties["Max Count"].apply(lambda x: 1 if not isinstance(x, int) or x == "" else x)
-
-    # Replace empty or non-string values in "Match Type" column with "exact"
-    properties["Match Type"] = properties["Match Type"].fillna("exact")
-    properties["Match Type"] = properties["Match Type"].apply(
-        lambda x: "exact" if not isinstance(x, str) or len(x) == 0 else x
-    )
-
-    # Replace empty or non-string values in "Comment" column with a default value
-    properties["Comment"] = properties["Comment"].fillna("Imported from Ontology by NEAT")
-    properties["Comment"] = properties["Comment"].apply(
-        lambda x: "Imported from Ontology by NEAT" if not isinstance(x, str) or len(x) == 0 else x
-    )
-
-    # Replace empty or non-boolean values in "Deprecated" column with False
-    properties["Deprecated"] = properties["Deprecated"].fillna(False)
-    properties["Deprecated"] = properties["Deprecated"].apply(lambda x: False if not isinstance(x, bool) else x)
-
-    # Reduce length of elements in the "Description" column to 1024 characters
-    properties["Description"] = properties["Description"].apply(lambda x: x[:1024] if isinstance(x, str) else None)
-
-    # fixes and additions
-    properties = fix_dangling_properties(properties)
-    properties = fix_missing_property_value_type(properties)
-
-    return properties
-
-
-def fix_dangling_properties(properties: pd.DataFrame) -> pd.DataFrame:
-    """This method fixes properties which are missing a domain definition in the ontology.
-
-    Args:
-        properties: Dataframe containing properties
-
-    Returns:
-        Dataframe containing properties with fixed domain
-    """
-    domain = {
-        "ObjectProperty": _object_property_class()["Class"],
-        "DatatypeProperty": _data_type_property_class()["Class"],
-    }
-
-    # apply missing range
-    properties["Class"] = properties.apply(
-        lambda row: domain[row._property_type]
-        if row._property_type == "ObjectProperty" and pd.isna(row["Class"])
-        else domain["DatatypeProperty"]
-        if pd.isna(row["Class"])
-        else row["Class"],
-        axis=1,
-    )
-    return properties
-
-
-def fix_missing_property_value_type(properties: pd.DataFrame) -> pd.DataFrame:
-    """This method fixes properties which are missing a range definition in the ontology.
-
-    Args:
-        properties: Dataframe containing properties
-
-    Returns:
-        Dataframe containing properties with fixed range
-    """
-    # apply missing range
-    properties["Type"] = properties.apply(
-        lambda row: _thing_class()["Class"]
-        if row._property_type == "ObjectProperty" and pd.isna(row["Type"])
-        else "string"
-        if pd.isna(row["Type"])
-        else row["Type"],
-        axis=1,
-    )
-
-    return properties

cognite/neat/legacy/rules/importers/_owl2rules/_owl2rules.py

@@ -1,290 +0,0 @@
-"""This module performs importing of various formats to one of serializations for which
-there are loaders to TransformationRules pydantic class."""
-
-# TODO: if this module grows too big, split it into several files and place under ./converter directory
-
-from pathlib import Path
-
-import pandas as pd
-from pydantic_core import ErrorDetails
-from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SKOS, Graph
-
-from cognite.neat.legacy.rules.importers._base import BaseImporter
-from cognite.neat.legacy.rules.models.raw_rules import RawRules
-from cognite.neat.legacy.rules.models.rules import Rules
-from cognite.neat.legacy.rules.models.tables import Tables
-from cognite.neat.legacy.rules.models.value_types import XSD_VALUE_TYPE_MAPPINGS
-
-from ._owl2classes import parse_owl_classes
-from ._owl2metadata import parse_owl_metadata
-from ._owl2properties import parse_owl_properties
-
-
-class OWLImporter(BaseImporter):
-    """Convert OWL ontology to tables/ transformation rules / Excel file.
-
-        Args:
-            owl_filepath: Path to OWL ontology
-
-    !!! Note
-        OWL Ontologies typically lacks some information that is required for making a complete
-        data model. This means that the methods .to_rules() will typically fail. Instead, it is recommended
-        that you use the .to_spreadsheet() method to generate an Excel file, and then manually add the missing
-        information to the Excel file. The Excel file can then be converted to a `Rules` object.
-
-        Alternatively, one can set the `make_compliant` parameter to True to allow neat to attempt to make
-        the imported rules compliant by adding default values for missing information, attaching dangling
-        properties to default containers based on the property type, etc. One has to be aware
-        that NEAT will be opinionated about how to make the ontology compliant, and that the resulting
-        rules may not be what you expect.
-
-    """
-
-    def __init__(self, owl_filepath: Path):
-        self.owl_filepath = owl_filepath
-
-    def to_tables(self, make_compliant: bool = False) -> dict[str, pd.DataFrame]:
-        graph = Graph()
-        try:
-            graph.parse(self.owl_filepath)
-        except Exception as e:
-            raise Exception(f"Could not parse owl file: {e}") from e
-
-        # bind key namespaces
-        graph.bind("owl", OWL)
-        graph.bind("rdf", RDF)
-        graph.bind("rdfs", RDFS)
-        graph.bind("dcterms", DCTERMS)
-        graph.bind("dc", DC)
-        graph.bind("skos", SKOS)
-
-        tables: dict[str, pd.DataFrame] = {
-            Tables.metadata: parse_owl_metadata(graph, make_compliant=make_compliant),
-            Tables.classes: parse_owl_classes(graph, make_compliant=make_compliant),
-            Tables.properties: parse_owl_properties(graph, make_compliant=make_compliant),
-        }
-
-        if make_compliant:
-            tables = make_tables_compliant(tables)
-        # add sorting of classes and properties prior exporting
-
-        tables[Tables.classes] = tables[Tables.classes].sort_values(by=["Class"])
-        tables[Tables.properties] = tables[Tables.properties].sort_values(by=["Class", "Property"])
-
-        return tables
-
-    def to_raw_rules(self, make_compliant: bool = False) -> RawRules:
-        """Creates `RawRules` object from the data."""
-
-        tables = self.to_tables(make_compliant=make_compliant)
-
-        return RawRules.from_tables(tables=tables, importer_type=self.__class__.__name__)
-
-    def to_rules(
-        self,
-        return_report: bool = False,
-        skip_validation: bool = False,
-        validators_to_skip: set[str] | None = None,
-        make_compliant: bool = False,
-    ) -> tuple[Rules | None, list[ErrorDetails] | None, list | None] | Rules:
-        """
-        Creates `Rules` object from the data.
-
-        Args:
-            return_report: To return validation report. Defaults to False.
-            skip_validation: Bypasses Rules validation. Defaults to False.
-            validators_to_skip: List of validators to skip. Defaults to None.
-            make_compliant: Flag for generating compliant rules, by default False
-
-        Returns:
-            Instance of `Rules`, which can be validated, not validated based on
-            `skip_validation` flag, or partially validated if `validators_to_skip` is set,
-            and optional list of errors and warnings if
-            `return_report` is set to True.
-
-        !!! Note "Skip Validation
-            `skip_validation` flag should be only used for purpose when `Rules` object
-            is exported to an Excel file. Do not use this flag for any other purpose!
-        """
-
-        raw_rules = self.to_raw_rules(make_compliant=make_compliant)
-
-        return raw_rules.to_rules(return_report, skip_validation, validators_to_skip)
-
-
-def make_tables_compliant(tables: dict[str, pd.DataFrame]) -> dict[str, pd.DataFrame]:
-    tables = _add_missing_classes(tables)
-    tables = _add_missing_value_types(tables)
-    tables = _add_properties_to_dangling_classes(tables)
-    tables = _add_entity_type_property(tables)
-
-    return tables
-
-
-def _add_missing_classes(tables: dict[str, pd.DataFrame]) -> dict[str, pd.DataFrame]:
-    """Add missing classes to containers.
-
-    Args:
-        tables: imported tables from owl ontology
-
-    Returns:
-        Updated tables with missing classes added to containers
-    """
-
-    missing_classes = set(tables[Tables.properties].Class.to_list()) - set(tables[Tables.classes].Class.to_list())
-
-    rows = []
-    for class_ in missing_classes:
-        rows += [
-            {
-                "Class": class_,
-                "Name": None,
-                "Description": None,
-                "Parent Class": None,
-                "Deprecated": False,
-                "Deprecation Date": None,
-                "Replaced By": None,
-                "Source": None,
-                "Source Entity Name": None,
-                "Match Type": None,
-                "Comment": (
-                    "Added by NEAT. "
-                    "This is a class that a domain of a property but was not defined in the ontology. "
-                    "It is added by NEAT to make the ontology compliant with CDF."
-                ),
-            }
-        ]
-
-    if rows:
-        tables[Tables.classes] = pd.concat(
-            [tables[Tables.classes], pd.DataFrame(rows)],
-            ignore_index=True,
-        )
-
-    return tables
-
-
-def _add_missing_value_types(tables: dict[str, pd.DataFrame]) -> dict[str, pd.DataFrame]:
-    """Add properties to classes that do not have any properties defined to them
-
-    Args:
-        tables: imported tables from owl ontology
-
-    Returns:
-        Updated tables with missing properties added to containers
-    """
-
-    xsd_types = set(XSD_VALUE_TYPE_MAPPINGS.keys())
-    referred_types = set(tables[Tables.properties]["Type"].to_list())
-    defined_classes = set(tables[Tables.classes]["Class"].to_list())
-
-    rows = []
-    for class_ in referred_types.difference(defined_classes).difference(xsd_types):
-        rows += [
-            {
-                "Class": class_,
-                "Name": None,
-                "Description": None,
-                "Parent Class": None,
-                "Deprecated": False,
-                "Deprecation Date": None,
-                "Replaced By": None,
-                "Source": None,
-                "Source Entity Name": None,
-                "Match Type": None,
-                "Comment": (
-                    "Added by NEAT. "
-                    "This is a class that a domain of a property but was not defined in the ontology. "
-                    "It is added by NEAT to make the ontology compliant with CDF."
-                ),
-            }
-        ]
-
-    if rows:
-        tables[Tables.classes] = pd.concat(
-            [tables[Tables.classes], pd.DataFrame(rows)],
-            ignore_index=True,
-        )
-
-    return tables
-
-
-def _add_properties_to_dangling_classes(
-    tables: dict[str, pd.DataFrame], properties_to_add: list[str] | None = None
-) -> dict[str, pd.DataFrame]:
-    """Add properties to classes that do not have any properties defined to them
-
-    Args:
-        tables: imported tables from owl ontology
-
-    Returns:
-        Updated tables with missing properties added to containers
-    """
-
-    if properties_to_add is None:
-        properties_to_add = ["label"]
-    undefined_classes = set(tables[Tables.classes].Class.to_list()) - set(tables[Tables.properties].Class.to_list())
-
-    rows = []
-    for class_ in undefined_classes:
-        for property_ in properties_to_add:
-            rows += [
-                {
-                    "Class": class_,
-                    "Property": property_,
-                    "Name": property_,
-                    "Description": None,
-                    "Type": "string",
-                    "Min Count": None,
-                    "Max Count": 1,
-                    "Deprecated": False,
-                    "Deprecation Date": None,
-                    "Replaced By": None,
-                    "Source": None,
-                    "Source Entity Name": None,
-                    "Match Type": None,
-                    "Comment": "Added by NEAT. Default property to make the ontology compliant with CDF.",
-                }
-            ]
-
-    if rows:
-        tables[Tables.properties] = pd.concat(
-            [tables[Tables.properties], pd.DataFrame(rows)],
-            ignore_index=True,
-        )
-
-    return tables
-
-
-def _add_entity_type_property(tables: dict[str, pd.DataFrame]) -> dict[str, pd.DataFrame]:
-    missing_entity_type = set(
-        tables[Tables.properties].groupby("Class").filter(lambda x: "entityType" not in x.Property.to_list()).Class
-    )
-
-    rows = []
-    for class_ in missing_entity_type:
-        rows += [
-            {
-                "Class": class_,
-                "Property": "entityType",
-                "Name": "entityType",
-                "Description": None,
-                "Type": "string",
-                "Min Count": None,
-                "Max Count": 1,
-                "Deprecated": False,
-                "Deprecation Date": None,
-                "Replaced By": None,
-                "Source": None,
-                "Source Entity Name": None,
-                "Match Type": None,
-                "Comment": "Added by NEAT. Default property added to make the ontology compliant with CDF.",
-            }
-        ]
-
-    if rows:
-        tables[Tables.properties] = pd.concat(
-            [tables[Tables.properties], pd.DataFrame(rows)],
-            ignore_index=True,
-        )
-    return tables

cognite/neat/legacy/rules/importers/_spreadsheet2rules.py

@@ -1,45 +0,0 @@
-"""This module performs importing of graph to TransformationRules pydantic class.
-In more details, it traverses the graph and abstracts class and properties, basically
-generating a list of rules based on which nodes that form the graph are made.
-"""
-
-from pathlib import Path
-
-import pandas as pd
-from openpyxl import Workbook, load_workbook
-
-from cognite.neat.utils.auxiliary import local_import
-
-from ._base import BaseImporter
-
-
-class ExcelImporter(BaseImporter):
-    def __init__(self, filepath: Path):
-        self.filepath = filepath
-
-    def to_tables(self) -> dict[str, pd.DataFrame]:
-        workbook: Workbook = load_workbook(self.filepath)
-
-        return {
-            sheet_name: pd.read_excel(
-                self.filepath,
-                sheet_name=sheet_name,
-                header=None if sheet_name == "Metadata" else 0,
-                skiprows=1 if sheet_name in ["Classes", "Properties", "Instances"] else None,
-            )
-            for sheet_name in workbook.sheetnames
-        }
-
-
-class GoogleSheetImporter(BaseImporter):
-    def __init__(self, sheet_id: str):
-        self.sheet_id = sheet_id
-
-    def to_tables(self) -> dict[str, pd.DataFrame]:
-        local_import("gspread", "google")
-        import gspread  # type: ignore[import]
-
-        client_google = gspread.service_account()
-        google_sheet = client_google.open_by_key(self.sheet_id)
-
-        return {worksheet.title: pd.DataFrame(worksheet.get_all_records()) for worksheet in google_sheet.worksheets()}

cognite/neat/legacy/rules/importers/_xsd2rules.py

@@ -1,20 +0,0 @@
-from pathlib import Path
-
-import pandas as pd
-
-from ._base import BaseImporter
-
-
-class XSDImporter(BaseImporter):
-    """
-    Importer for XSD (XML Schema) files.
-
-    Args:
-        xml_directory: Path to directory containing XSD files.
-    """
-
-    def __init__(self, xsd_directory: Path):
-        self.xsd_directory = xsd_directory
-
-    def to_tables(self) -> dict[str, pd.DataFrame]:
-        raise NotImplementedError

cognite/neat/legacy/rules/importers/_yaml2rules.py

@@ -1,39 +0,0 @@
-from pathlib import Path
-from typing import Literal
-
-import yaml
-
-from ._dict2rules import ArbitraryDictImporter
-
-
-class ArbitraryYAMLImporter(ArbitraryDictImporter):
-    """
-    Importer for data given in a YAML file or string.
-
-    This importer infers the data model from the YAML string based on the shape of the data.
-
-    Args:
-        yaml_path_or_str: Path to file with YAML.
-        relationship_direction: Direction of relationships, either "parent-to-child" or "child-to-parent". JSON
-            files are nested with children nested inside parents. This option determines whether the resulting rules
-            will have an edge from parents to children or from children to parents.
-    """
-
-    def __init__(
-        self,
-        yaml_path_or_str: Path,
-        relationship_direction: Literal["parent-to-child", "child-to-parent"] = "parent-to-child",
-    ):
-        if isinstance(yaml_path_or_str, str):
-            data = yaml.safe_load(yaml_path_or_str)
-            super().__init__(data, relationship_direction)
-        elif isinstance(yaml_path_or_str, Path):
-            if not yaml_path_or_str.exists():
-                raise ValueError(f"File {yaml_path_or_str} does not exist")
-            if yaml_path_or_str.suffix != ".json":
-                raise ValueError(f"File {yaml_path_or_str} is not a JSON file")
-            self.json_path = yaml_path_or_str
-            data = yaml.safe_load(yaml_path_or_str.read_text())
-            super().__init__(data, relationship_direction)
-        else:
-            raise TypeError(f"Expected Path or str, got {type(yaml_path_or_str)}")

cognite/neat/legacy/rules/models/__init__.py

@@ -1,5 +0,0 @@
-from .raw_rules import RawRules
-from .rules import Rules
-from .tables import Tables
-
-__all__ = ["Tables", "Rules", "RawRules"]