cognite-neat 0.87.6__py3-none-any.whl → 0.88.0__py3-none-any.whl

cognite/neat/legacy/rules/exporters/_rules2rules.py

@@ -1,105 +0,0 @@
-"""This module provides set of methods that perform conversion of TransformationRules
-to TransformationRules for purpose of for example:
-
-- subsetting the data model to only include desired classes and their properties
-- converting classes/properties ids/names to DMS compliant format
-"""
-
-import logging
-import re
-import warnings
-from typing import Any
-
-from cognite.neat.legacy.rules.analysis import get_defined_classes
-from cognite.neat.legacy.rules.models.rules import Rules
-
-
-def subset_rules(rules: Rules, desired_classes: set, skip_validation: bool = False) -> Rules:
-    """
-    Subset transformation rules to only include desired classes and their properties.
-
-    Args:
-        transformation_rules: Instance of TransformationRules to subset
-        desired_classes: Desired classes to include in the reduced data model
-        skip_validation: Whether to skip underlying pydantic validation, by default False
-
-    Returns:
-        Instance of TransformationRules
-
-    !!! note "Skipping Validation"
-        It is fine to skip validation since we are deriving the reduced data model from data
-        model (i.e. TransformationRules) which has already been validated.
-
-    """
-
-    defined_classes = get_defined_classes(rules)
-    possible_classes = defined_classes.intersection(desired_classes)
-    impossible_classes = desired_classes - possible_classes
-
-    if not possible_classes:
-        logging.error("None of the desired classes are defined in the data model!")
-        raise ValueError("None of the desired classes are defined in the data model!")
-
-    if impossible_classes:
-        logging.warning(f"Could not find the following classes defined in the data model: {impossible_classes}")
-        warnings.warn(
-            f"Could not find the following classes defined in the data model: {impossible_classes}", stacklevel=2
-        )
-
-    reduced_data_model: dict[str, Any] = {
-        "metadata": rules.metadata.model_copy(),
-        "prefixes": (rules.prefixes or {}).copy(),
-        "classes": {},
-        "properties": {},
-        "instances": (rules.instances or []).copy(),
-    }
-
-    logging.info(f"Reducing data model to only include the following classes: {possible_classes}")
-    for class_ in possible_classes:
-        reduced_data_model["classes"][class_] = rules.classes[class_]
-
-    for id_, property_definition in rules.properties.items():
-        if property_definition.class_id in possible_classes:
-            reduced_data_model["properties"][id_] = property_definition
-
-    if skip_validation:
-        return Rules.model_construct(**reduced_data_model)
-    else:
-        return Rules(**reduced_data_model)
-
-
-def to_dms_compliant_rules(rules: Rules) -> Rules:
-    raise NotImplementedError()
-
-
-# to be used for conversion to DMS compliant format
-def to_dms_name(name: str, entity_type: str, fix_casing: bool = False) -> str:
-    """
-    Repairs an entity name to conform to GraphQL naming convention
-    >>> repair_name("wind-speed", "property")
-    'windspeed'
-    >>> repair_name("Wind.Speed", "property", True)
-    'windSpeed'
-    >>> repair_name("windSpeed", "class", True)
-    'WindSpeed'
-    >>> repair_name("22windSpeed", "class")
-    '_22windSpeed'
-    """
-
-    # Remove any non GraphQL compliant characters
-    repaired_string = re.sub(r"[^_a-zA-Z0-9]", "", name)
-
-    # Name must start with a letter or underscore
-    if repaired_string[0].isdigit():
-        repaired_string = f"_{repaired_string}"
-
-    if not fix_casing:
-        return repaired_string
-    # Property names must be camelCase
-    if entity_type == "property" and repaired_string[0].isupper():
-        return repaired_string[0].lower() + repaired_string[1:]
-    # Class names must be PascalCase
-    elif entity_type == "class" and repaired_string[0].islower():
-        return repaired_string[0].upper() + repaired_string[1:]
-    else:
-        return repaired_string

cognite/neat/legacy/rules/exporters/_rules2triples.py

@@ -1,38 +0,0 @@
-from pathlib import Path
-
-from rdflib.term import Node
-
-from cognite.neat.legacy.rules.models.rules import Rules
-
-from ._base import BaseExporter
-
-
-class TripleExporter(BaseExporter[list[tuple[Node, Node, Node]]]):
-    """
-    Exporter for transformation rules instances sheet to RDF triples
-    """
-
-    def _export_to_file(self, filepath: Path) -> None:
-        raise NotImplementedError("Export to file not implemented")
-
-    def export(self) -> list[tuple[Node, Node, Node]]:
-        return get_instances_as_triples(self.rules)
-
-
-def get_instances_as_triples(transformation_rules: Rules) -> list[tuple[Node, Node, Node]]:
-    """
-    Converts transformation rules instances sheet to RDF triples
-
-    Args:
-        transformation_rules: An instance of TransformationRules pydantic class
-
-    Returns:
-        List of triples provided as tuples
-
-    """
-    if transformation_rules.instances:
-        return [
-            (instance.instance, instance.property_, instance.value)  # type: ignore[misc]
-            for instance in transformation_rules.instances
-        ]
-    return []

cognite/neat/legacy/rules/exporters/_validation.py

@@ -1,146 +0,0 @@
-import re
-import warnings
-from typing import Literal, overload
-
-from cognite.neat.exceptions import wrangle_warnings
-from cognite.neat.legacy.rules import exceptions
-from cognite.neat.legacy.rules.models.rules import (
-    Rules,
-    dms_property_id_compliance_regex,
-    value_id_compliance_regex,
-    view_id_compliance_regex,
-)
-
-
-@overload
-def are_entity_names_dms_compliant(
-    transformation_rules: Rules, return_report: Literal[True]
-) -> tuple[bool, list[dict]]: ...
-
-
-@overload
-def are_entity_names_dms_compliant(transformation_rules: Rules, return_report: Literal[False] = False) -> bool: ...
-
-
-def are_entity_names_dms_compliant(
-    transformation_rules: Rules, return_report: bool = False
-) -> bool | tuple[bool, list[dict]]:
-    """Check if data model definitions are valid."""
-
-    flag: bool = True
-    with warnings.catch_warnings(record=True) as validation_warnings:
-        for class_ in transformation_rules.classes.values():
-            if not re.match(view_id_compliance_regex, class_.class_id):
-                warnings.warn(
-                    exceptions.EntityIDNotDMSCompliant(
-                        "Class", class_.class_id, f"[Classes/Class/{class_.class_id}]"
-                    ).message,
-                    category=exceptions.EntityIDNotDMSCompliant,
-                    stacklevel=2,
-                )
-                flag = False
-
-        for row, property_ in transformation_rules.properties.items():
-            # check class id which would resolve as view/container id
-            if not re.match(view_id_compliance_regex, property_.class_id):
-                warnings.warn(
-                    exceptions.EntityIDNotDMSCompliant(
-                        "Class", property_.class_id, f"[Properties/Class/{row}]"
-                    ).message,
-                    category=exceptions.EntityIDNotDMSCompliant,
-                    stacklevel=2,
-                )
-                flag = False
-
-            # check property id which would resolve as view/container id
-            if not re.match(dms_property_id_compliance_regex, property_.property_id):
-                warnings.warn(
-                    exceptions.EntityIDNotDMSCompliant(
-                        "Property", property_.property_id, f"[Properties/Property/{row}]"
-                    ).message,
-                    category=exceptions.EntityIDNotDMSCompliant,
-                    stacklevel=2,
-                )
-                flag = False
-
-            # check container external id
-            if property_.container and not re.match(view_id_compliance_regex, property_.container.external_id):
-                warnings.warn(
-                    exceptions.EntityIDNotDMSCompliant(
-                        "Container", property_.container.external_id, f"[Properties/Container/{row}]"
-                    ).message,
-                    category=exceptions.EntityIDNotDMSCompliant,
-                    stacklevel=2,
-                )
-                flag = False
-
-            # check container property external id
-            if property_.container_property and not re.match(
-                dms_property_id_compliance_regex, property_.container_property
-            ):
-                warnings.warn(
-                    exceptions.EntityIDNotDMSCompliant(
-                        "Container Property", property_.container_property, f"[Properties/Container Property/{row}]"
-                    ).message,
-                    category=exceptions.EntityIDNotDMSCompliant,
-                    stacklevel=2,
-                )
-                flag = False
-
-            # expected value type, as it is case sensitive should be ok
-            if not re.match(value_id_compliance_regex, property_.expected_value_type.external_id):
-                warnings.warn(
-                    exceptions.EntityIDNotDMSCompliant(
-                        "Value type", property_.expected_value_type.external_id, f"[Properties/Type/{row}]"
-                    ).message,
-                    category=exceptions.EntityIDNotDMSCompliant,
-                    stacklevel=2,
-                )
-                flag = False
-
-    if return_report:
-        return flag, wrangle_warnings(validation_warnings)
-    else:
-        return flag
-
-
-@overload
-def are_properties_redefined(transformation_rules: Rules, return_report: Literal[True]) -> tuple[bool, list[dict]]: ...
-
-
-@overload
-def are_properties_redefined(transformation_rules: Rules, return_report: Literal[False] = False) -> bool: ...
-
-
-def are_properties_redefined(
-    transformation_rules: Rules, return_report: bool = False
-) -> bool | tuple[bool, list[dict]]:
-    flag: bool = False
-    with warnings.catch_warnings(record=True) as validation_warnings:
-        analyzed_properties = {}
-        for property_ in transformation_rules.properties.values():
-            if property_.property_id not in analyzed_properties:
-                analyzed_properties[property_.property_id] = [property_.class_id]
-            elif property_.class_id in analyzed_properties[property_.property_id]:
-                flag = True
-                warnings.warn(
-                    exceptions.PropertyRedefined(property_.property_id, property_.class_id).message,
-                    category=exceptions.EntityIDNotDMSCompliant,
-                    stacklevel=2,
-                )
-
-            else:
-                analyzed_properties[property_.property_id].append(property_.class_id)
-
-    if return_report:
-        return flag, wrangle_warnings(validation_warnings)
-    else:
-        return flag
-
-
-def property_ids_camel_case_compliant(transformation_rules) -> bool | tuple[bool, list[dict]]:
-    raise NotImplementedError()
-
-
-def class_id_pascal_case_compliant(transformation_rules) -> bool | tuple[bool, list[dict]]:
-    raise NotImplementedError()

cognite/neat/legacy/rules/importers/__init__.py

@@ -1,22 +0,0 @@
-from ._base import BaseImporter
-from ._dict2rules import ArbitraryDictImporter
-from ._dms2rules import DMSImporter
-from ._graph2rules import GraphImporter
-from ._json2rules import ArbitraryJSONImporter
-from ._owl2rules import OWLImporter
-from ._spreadsheet2rules import ExcelImporter, GoogleSheetImporter
-from ._xsd2rules import XSDImporter
-from ._yaml2rules import ArbitraryYAMLImporter
-
-__all__ = [
-    "BaseImporter",
-    "ArbitraryDictImporter",
-    "ArbitraryJSONImporter",
-    "ArbitraryYAMLImporter",
-    "DMSImporter",
-    "OWLImporter",
-    "XSDImporter",
-    "GraphImporter",
-    "ExcelImporter",
-    "GoogleSheetImporter",
-]

cognite/neat/legacy/rules/importers/_base.py

@@ -1,66 +0,0 @@
-import getpass
-from abc import ABC, abstractmethod
-from datetime import datetime
-
-import pandas as pd
-from pydantic_core import ErrorDetails
-
-from cognite.neat.legacy.rules.models.raw_rules import RawRules
-from cognite.neat.legacy.rules.models.rules import Rules
-
-
-class BaseImporter(ABC):
-    """
-    BaseImporter class which all importers inherit from.
-    """
-
-    @abstractmethod
-    def to_tables(self) -> dict[str, pd.DataFrame]:
-        """Creates raw tables from the data."""
-        raise NotImplementedError
-
-    def to_raw_rules(self) -> RawRules:
-        """Creates `RawRules` object from the data."""
-
-        tables = self.to_tables()
-
-        return RawRules.from_tables(tables=tables, importer_type=self.__class__.__name__)
-
-    def to_rules(
-        self,
-        return_report: bool = False,
-        skip_validation: bool = False,
-        validators_to_skip: set[str] | None = None,
-    ) -> tuple[Rules | None, list[ErrorDetails] | None, list | None] | Rules:
-        """
-        Creates `Rules` object from the data.
-
-        Args:
-            return_report: To return validation report. Defaults to False.
-            skip_validation: Bypasses Rules validation. Defaults to False.
-            validators_to_skip: List of validators to skip. Defaults to None.
-
-        Returns:
-            Instance of `Rules`, which can be validated, not validated based on
-            `skip_validation` flag, or partially validated if `validators_to_skip` is set,
-            and optional list of errors and warnings if
-            `return_report` is set to True.
-
-        !!! Note "Skip Validation
-            `skip_validation` flag should be only used for purpose when `Rules` object
-            is exported to an Excel file. Do not use this flag for any other purpose!
-        """
-
-        raw_rules = self.to_raw_rules()
-
-        return raw_rules.to_rules(return_report, skip_validation, validators_to_skip)
-
-    def _default_metadata(self):
-        return {
-            "prefix": "neat",
-            "version": "0.1.0",
-            "title": "Neat Imported Data Model",
-            "created": datetime.now().replace(microsecond=0).isoformat(),
-            "creator": getpass.getuser(),
-            "description": f"Imported using {type(self).__name__}",
-        }

cognite/neat/legacy/rules/importers/_dict2rules.py

@@ -1,158 +0,0 @@
-from datetime import datetime, timezone
-from typing import Any, Literal
-
-import pandas as pd
-
-from cognite.neat.legacy.rules.models.tables import Tables
-
-from ._base import BaseImporter
-
-
-class ArbitraryDictImporter(BaseImporter):
-    """
-    Importer for an arbitrary dictionary.
-
-    This importer infers the data model from the dictionary based on the shape of the data.
-
-    Args:
-        data: dictionary containing Rules definitions.
-        relationship_direction: Direction of relationships, either "parent-to-child" or "child-to-parent". Dictionaries
-            are nested with children nested inside parents. This option determines whether the resulting rules
-            will have an edge from parents to children or from children to parents.
-    """
-
-    def __init__(
-        self,
-        data: dict[str, Any],
-        relationship_direction: Literal["parent-to-child", "child-to-parent"] = "parent-to-child",
-    ):
-        self.data = data
-        self.relationship_direction = relationship_direction
-
-    def to_tables(self) -> dict[str, pd.DataFrame]:
-        metadata = pd.Series(
-            dict(
-                title="OpenAPI to DM transformation rules",
-                description="OpenAPI to DM transformation rules",
-                version="0.1",
-                creator="Cognite",
-                created=datetime.now(timezone.utc).replace(microsecond=0).isoformat(),
-                namespace="http://purl.org/cognite/neat#",
-                prefix="neat",
-                data_model_name="OpenAPI",
-                cdf_space_name="OpenAPI",
-            )
-        ).reset_index()
-        finder = _TripleFinder(self.relationship_direction)
-        finder.find_triples(self.data)
-
-        return {
-            Tables.metadata: metadata,
-            Tables.classes: pd.DataFrame(finder.classes).T,
-            Tables.properties: pd.DataFrame(finder.properties).T,
-        }
-
-
-class _TripleFinder:
-    def __init__(self, relationship_direction: Literal["parent-to-child", "child-to-parent"]) -> None:
-        self.classes: dict[str, dict[str, Any]] = {}
-        self.properties: dict[str, dict[str, Any]] = {}
-        self.relationship_direction = relationship_direction
-
-    def find_triples(self, data: dict[str, Any]) -> None:
-        self._convert_dict_to_classes_and_props(data)
-
-    def _convert_dict_to_classes_and_props(
-        self, data: dict, parent_property_name: str | None = None, grand_parent_property_name: str | None = None
-    ) -> None:
-        if isinstance(data, dict) and len(data) == 0:
-            return
-        elif isinstance(data, dict) and parent_property_name is None:
-            for key, value in data.items():
-                self._convert_dict_to_classes_and_props(value, key)
-        elif isinstance(data, dict):
-            self.add_class(parent_property_name, "missing", grand_parent_property_name, is_list=False)
-            for key, value in data.items():
-                self._convert_dict_to_classes_and_props(value, key, parent_property_name)
-        elif isinstance(data, list):
-            if parent_property_name is not None and grand_parent_property_name is not None:
-                data_type = self._get_list_type(data, parent_property_name)
-                self.add_property(grand_parent_property_name, parent_property_name, data_type, "missing", is_list=True)
-            for item in data:
-                self._convert_dict_to_classes_and_props(item, parent_property_name, grand_parent_property_name)
-        elif isinstance(data, bool | int | float | str) and parent_property_name is not None:
-            data_type = self._get_primitive_data_type(data)
-            self.add_property(grand_parent_property_name, parent_property_name, data_type, "missing")
-        else:
-            raise ValueError(f"Unknown type {type(data)}")
-
-    def add_class(
-        self, class_name: str, description: str = "", parent_class_name: str | None = None, is_list: bool = False
-    ):
-        if class_name in self.classes:
-            return
-        class_ = {"Class": class_name, "description": description}
-        if parent_class_name:
-            if self.relationship_direction == "child-to-parent":
-                self.add_property(class_name, "parent", parent_class_name, "missing", is_list=False)
-            elif self.relationship_direction == "parent-to-child":
-                self.add_property(parent_class_name, class_name, class_name, "missing", is_list)
-            else:
-                raise ValueError(f"Unknown relationship direction {self.relationship_direction}")
-        self.classes[class_name] = class_
-
-    def add_property(
-        self,
-        class_name: str,
-        property_name: str,
-        property_type: str,
-        description: str = "missing",
-        is_list: bool = False,
-    ):
-        if class_name + property_name in self.properties:
-            return
-        prop = dict(
-            class_id=class_name,
-            property_id=property_name,
-            property_name=property_name,
-            property_type="ObjectProperty",
-            description=description,
-            expected_value_type=property_type,
-            max_count=1 if not is_list else None,
-            cdf_resource_type="Asset",
-            resource_type_property="Asset",
-            rule_type="rdfpath",
-            rule=f"neat:{class_name}(neat:{property_name})",
-            label="linked to",
-        )
-        self.properties[class_name + property_name] = prop
-
-    @staticmethod
-    def _get_primitive_data_type(data: Any, errors: Literal["raise", "empty"] = "raise") -> str:
-        data_type = type(data)
-        if data_type is bool:
-            return "boolean"
-        elif data_type is int:
-            return "integer"
-        elif data_type is float:
-            return "float"
-        elif data_type is str and not pd.isna(pd.to_datetime(data, errors="coerce")):
-            return "dateTime"
-        elif data_type is str:
-            return "string"
-
-        if errors == "empty":
-            return ""
-        else:
-            raise ValueError(f"Unknown primitive type {data_type}")
-
-    @classmethod
-    def _get_list_type(cls, data: list[Any], class_name: str) -> str:
-        if isinstance(data[0], dict):
-            return class_name
-
-        data_types = {cls._get_primitive_data_type(item, "empty") for item in data}
-        if "" in data_types or len(data_types) > 1:
-            # Fallback to string
-            return "string"
-        return data_types.pop()