cognite-neat 0.86.0__py3-none-any.whl → 0.87.3__py3-none-any.whl

cognite/neat/rules/analysis/_information_rules.py

@@ -1,476 +0,0 @@
-import itertools
-import logging
-import warnings
-from collections import defaultdict
-from typing import Any, Generic, TypeVar
-
-import pandas as pd
-from pydantic import ValidationError
-
-from cognite.neat.rules.models import SchemaCompleteness
-from cognite.neat.rules.models._rdfpath import Hop, RDFPath
-from cognite.neat.rules.models.asset import AssetClass, AssetProperty, AssetRules
-from cognite.neat.rules.models.entities import (
-    AssetEntity,
-    ClassEntity,
-    EntityTypes,
-    ParentClassEntity,
-    ReferenceEntity,
-    RelationshipEntity,
-)
-from cognite.neat.rules.models.information import (
-    InformationClass,
-    InformationProperty,
-    InformationRules,
-)
-from cognite.neat.utils.utils import get_inheritance_path
-
-T_Rules = TypeVar("T_Rules", InformationRules, AssetRules)
-T_Property = TypeVar("T_Property", InformationProperty, AssetProperty)
-T_Class = TypeVar("T_Class", InformationClass, AssetClass)
-
-
-class _SharedAnalysis(Generic[T_Rules, T_Property, T_Class]):
-    def __init__(self, rules: T_Rules):
-        self.rules: T_Rules = rules
-
-    @property
-    def directly_referred_classes(self) -> set[ClassEntity]:
-        return {
-            class_.reference.as_class_entity()
-            for class_ in self.rules.classes
-            if self.rules.reference
-            and class_.reference
-            and isinstance(class_.reference, ReferenceEntity)
-            and class_.reference.prefix == self.rules.reference.metadata.prefix
-        }
-
-    @property
-    def inherited_referred_classes(self) -> set[ClassEntity]:
-        dir_referred_classes = self.directly_referred_classes
-        inherited_referred_classes = []
-        for class_ in dir_referred_classes:
-            inherited_referred_classes.extend(self.class_inheritance_path(class_))
-        return set(inherited_referred_classes)
-
-    def class_parent_pairs(self) -> dict[ClassEntity, list[ParentClassEntity]]:
-        """This only returns class - parent pairs only if parent is in the same data model"""
-        class_subclass_pairs: dict[ClassEntity, list[ParentClassEntity]] = {}
-
-        if not self.rules:
-            return class_subclass_pairs
-
-        for definition in self.rules.classes:
-            class_subclass_pairs[definition.class_] = []
-
-            if definition.parent is None:
-                continue
-
-            for parent in definition.parent:
-                if parent.prefix == definition.class_.prefix:
-                    class_subclass_pairs[definition.class_].append(parent)
-                else:
-                    warnings.warn(
-                        f"Parent class {parent} of class {definition} is not in the same namespace, skipping !",
-                        stacklevel=2,
-                    )
-
-        return class_subclass_pairs
-
-    def classes_with_properties(self, consider_inheritance: bool = False) -> dict[ClassEntity, list[T_Property]]:
-        """Returns classes that have been defined in the data model.
-
-        Args:
-            consider_inheritance: Whether to consider inheritance or not. Defaults False
-
-        Returns:
-            Dictionary of classes with a list of properties defined for them
-
-        !!! note "consider_inheritance"
-            If consider_inheritance is True, properties from parent classes will also be considered.
-            This means if a class has a parent class, and the parent class has properties defined for it,
-            while we do not have any properties defined for the child class, we will still consider the
-            properties from the parent class. If consider_inheritance is False, we will only consider
-            properties defined for the child class, thus if no properties are defined for the child class,
-            it will not be included in the returned dictionary.
-        """
-
-        class_property_pairs: dict[ClassEntity, list[T_Property]] = defaultdict(list)
-
-        for property_ in self.rules.properties:
-            class_property_pairs[property_.class_].append(property_)  # type: ignore
-
-        if consider_inheritance:
-            class_parent_pairs = self.class_parent_pairs()
-            for class_ in class_parent_pairs:
-                self._add_inherited_properties(class_, class_property_pairs, class_parent_pairs)
-
-        return class_property_pairs
-
-    def class_inheritance_path(self, class_: ClassEntity | str) -> list[ClassEntity]:
-        class_ = class_ if isinstance(class_, ClassEntity) else ClassEntity.load(class_)
-        class_parent_pairs = self.class_parent_pairs()
-        return get_inheritance_path(class_, class_parent_pairs)
-
-    @classmethod
-    def _add_inherited_properties(
-        cls,
-        class_: ClassEntity,
-        class_property_pairs: dict[ClassEntity, list[T_Property]],
-        class_parent_pairs: dict[ClassEntity, list[ParentClassEntity]],
-    ):
-        inheritance_path = get_inheritance_path(class_, class_parent_pairs)
-        for parent in inheritance_path:
-            # ParentClassEntity -> ClassEntity to match the type of class_property_pairs
-            if parent.as_class_entity() in class_property_pairs:
-                for property_ in class_property_pairs[parent.as_class_entity()]:
-                    property_ = property_.model_copy()
-
-                    # This corresponds to importing properties from parent class
-                    # making sure that the property is attached to desired child class
-                    property_.class_ = class_
-                    property_.inherited = True
-
-                    # need same if we have RDF path to make sure that the starting class is the
-
-                    if class_ in class_property_pairs:
-                        class_property_pairs[class_].append(property_)
-                    else:
-                        class_property_pairs[class_] = [property_]
-
-    def class_property_pairs(
-        self, only_rdfpath: bool = False, consider_inheritance: bool = False
-    ) -> dict[ClassEntity, dict[str, T_Property]]:
-        """Returns a dictionary of classes with a dictionary of properties associated with them.
-
-        Args:
-            only_rdfpath : To consider only properties which have rule `rdfpath` set. Defaults False
-            consider_inheritance: Whether to consider inheritance or not. Defaults False
-
-        Returns:
-            Dictionary of classes with a dictionary of properties associated with them.
-
-        !!! note "difference to get_classes_with_properties"
-            This method returns a dictionary of classes with a dictionary of properties associated with them.
-            While get_classes_with_properties returns a dictionary of classes with a list of
-            properties defined for them,
-            here we filter the properties based on the `only_rdfpath` parameter and only consider
-            the first definition of a property if it is defined more than once.
-
-        !!! note "only_rdfpath"
-            If only_rdfpath is True, only properties with RuleType.rdfpath will be returned as
-            a part of the dictionary of properties related to a class. Otherwise, all properties
-            will be returned.
-
-        !!! note "consider_inheritance"
-            If consider_inheritance is True, properties from parent classes will also be considered.
-            This means if a class has a parent class, and the parent class has properties defined for it,
-            while we do not have any properties defined for the child class, we will still consider the
-            properties from the parent class. If consider_inheritance is False, we will only consider
-            properties defined for the child class, thus if no properties are defined for the child class,
-            it will not be included in the returned dictionary.
-        """
-        # TODO: https://cognitedata.atlassian.net/jira/software/projects/NEAT/boards/893?selectedIssue=NEAT-78
-
-        class_property_pairs = {}
-
-        for class_, properties in self.classes_with_properties(consider_inheritance).items():
-            processed_properties = {}
-            for property_ in properties:
-                if property_.property_ in processed_properties:
-                    # TODO: use appropriate Warning class from _exceptions.py
-                    # if missing make one !
-                    warnings.warn(
-                        f"Property {property_.property_} for {class_} has been defined more than once!"
-                        " Only the first definition will be considered, skipping the rest..",
-                        stacklevel=2,
-                    )
-                    continue
-
-                if (only_rdfpath and isinstance(property_.transformation, RDFPath)) or not only_rdfpath:
-                    processed_properties[property_.property_] = property_
-            class_property_pairs[class_] = processed_properties
-
-        return class_property_pairs
-
-    def class_linkage(self, consider_inheritance: bool = False) -> pd.DataFrame:
-        """Returns a dataframe with the class linkage of the data model.
-
-        Args:
-            consider_inheritance: Whether to consider inheritance or not. Defaults False
-
-        Returns:
-            Dataframe with the class linkage of the data model
-        """
-
-        class_linkage = pd.DataFrame(
-            columns=[
-                "source_class",
-                "target_class",
-                "connecting_property",
-                "max_occurrence",
-            ]
-        )
-
-        class_property_pairs = self.classes_with_properties(consider_inheritance)
-        properties = list(itertools.chain.from_iterable(class_property_pairs.values()))
-
-        for property_ in properties:
-            if property_.type_ == EntityTypes.object_property:
-                new_row = pd.Series(
-                    {
-                        "source_class": property_.class_,
-                        "connecting_property": property_.property_,
-                        "target_class": property_.value_type,
-                        "max_occurrence": property_.max_count,
-                    }
-                )
-                class_linkage = pd.concat([class_linkage, new_row.to_frame().T], ignore_index=True)
-
-        class_linkage.drop_duplicates(inplace=True)
-        class_linkage = class_linkage[["source_class", "connecting_property", "target_class", "max_occurrence"]]
-
-        return class_linkage
-
-    def connected_classes(self, consider_inheritance: bool = False) -> set[ClassEntity]:
-        """Return a set of classes that are connected to other classes.
-
-        Args:
-            consider_inheritance: Whether to consider inheritance or not. Defaults False
-
-        Returns:
-            Set of classes that are connected to other classes
-        """
-        class_linkage = self.class_linkage(consider_inheritance)
-        return set(class_linkage.source_class.values).union(set(class_linkage.target_class.values))
-
-    def defined_classes(self, consider_inheritance: bool = False) -> set[ClassEntity]:
-        """Returns classes that have properties defined for them in the data model.
-
-        Args:
-            consider_inheritance: Whether to consider inheritance or not. Defaults False
-
-        Returns:
-            Set of classes that have been defined in the data model
-        """
-        class_property_pairs = self.classes_with_properties(consider_inheritance)
-        properties = list(itertools.chain.from_iterable(class_property_pairs.values()))
-
-        return {property.class_ for property in properties}
-
-    def disconnected_classes(self, consider_inheritance: bool = False) -> set[ClassEntity]:
-        """Return a set of classes that are disconnected (i.e. isolated) from other classes.
-
-        Args:
-            consider_inheritance: Whether to consider inheritance or not. Defaults False
-
-        Returns:
-            Set of classes that are disconnected from other classes
-        """
-        return self.defined_classes(consider_inheritance) - self.connected_classes(consider_inheritance)
-
-    def symmetrically_connected_classes(
-        self, consider_inheritance: bool = False
-    ) -> set[tuple[ClassEntity, ClassEntity]]:
-        """Returns a set of pairs of symmetrically linked classes.
-
-        Args:
-            consider_inheritance: Whether to consider inheritance or not. Defaults False
-
-        Returns:
-            Set of pairs of symmetrically linked classes
-
-        !!! note "Symmetrically Connected Classes"
-            Symmetrically connected classes are classes that are connected to each other
-            in both directions. For example, if class A is connected to class B, and class B
-            is connected to class A, then classes A and B are symmetrically connected.
-        """
-
-        # TODO: Find better name for this method
-        sym_pairs: set[tuple[ClassEntity, ClassEntity]] = set()
-
-        class_linkage = self.class_linkage(consider_inheritance)
-        if class_linkage.empty:
-            return sym_pairs
-
-        for _, row in class_linkage.iterrows():
-            source = row.source_class
-            target = row.target_class
-            target_targets = class_linkage[class_linkage.source_class == target].target_class.values
-            if source in target_targets and (source, target) not in sym_pairs:
-                sym_pairs.add((source, target))
-        return sym_pairs
-
-    def as_property_dict(
-        self,
-    ) -> dict[str, list[T_Property]]:
-        """This is used to capture all definitions of a property in the data model."""
-        property_dict: dict[str, list[T_Property]] = defaultdict(list)
-        for definition in self.rules.properties:
-            property_dict[definition.property_].append(definition)  # type: ignore
-        return property_dict
-
-    def as_class_dict(self) -> dict[str, T_Class]:
-        """This is to simplify access to classes through dict."""
-        class_dict: dict[str, T_Class] = {}
-        for definition in self.rules.classes:
-            class_dict[str(definition.class_.suffix)] = definition  # type: ignore
-        return class_dict
-
-    def subset_rules(self, desired_classes: set[ClassEntity]) -> T_Rules:
-        """
-        Subset rules to only include desired classes and their properties.
-
-        Args:
-            desired_classes: Desired classes to include in the reduced data model
-
-        Returns:
-            Instance of InformationRules
-
-        !!! note "Inheritance"
-            If desired classes contain a class that is a subclass of another class(es), the parent class(es)
-            will be included in the reduced data model as well even though the parent class(es) are
-            not in the desired classes set. This is to ensure that the reduced data model is
-            consistent and complete.
-
-        !!! note "Partial Reduction"
-            This method does not perform checks if classes that are value types of desired classes
-            properties are part of desired classes. If a class is not part of desired classes, but it
-            is a value type of a property of a class that is part of desired classes, derived reduced
-            rules will be marked as partial.
-
-        !!! note "Validation"
-            This method will attempt to validate the reduced rules with custom validations.
-            If it fails, it will return a partial rules with a warning message, validated
-            only with base Pydantic validators.
-        """
-
-        if self.rules.metadata.schema_ is not SchemaCompleteness.complete:
-            raise ValueError("Rules are not complete cannot perform reduction!")
-        class_as_dict = self.as_class_dict()
-        class_parents_pairs = self.class_parent_pairs()
-        defined_classes = self.defined_classes(consider_inheritance=True)
-
-        possible_classes = defined_classes.intersection(desired_classes)
-        impossible_classes = desired_classes - possible_classes
-
-        # need to add all the parent classes of the desired classes to the possible classes
-        parents: set[ClassEntity] = set()
-        for class_ in possible_classes:
-            parents = parents.union(
-                {parent.as_class_entity() for parent in get_inheritance_path(class_, class_parents_pairs)}
-            )
-        possible_classes = possible_classes.union(parents)
-
-        if not possible_classes:
-            logging.error("None of the desired classes are defined in the data model!")
-            raise ValueError("None of the desired classes are defined in the data model!")
-
-        if impossible_classes:
-            logging.warning(f"Could not find the following classes defined in the data model: {impossible_classes}")
-            warnings.warn(
-                f"Could not find the following classes defined in the data model: {impossible_classes}",
-                stacklevel=2,
-            )
-
-        reduced_data_model: dict[str, Any] = {
-            "metadata": self.rules.metadata.model_copy(),
-            "prefixes": (self.rules.prefixes or {}).copy(),
-            "classes": [],
-            "properties": [],
-        }
-
-        logging.info(f"Reducing data model to only include the following classes: {possible_classes}")
-        for class_ in possible_classes:
-            reduced_data_model["classes"].append(class_as_dict[str(class_.suffix)])
-
-        class_property_pairs = self.classes_with_properties(consider_inheritance=False)
-
-        for class_, properties in class_property_pairs.items():
-            if class_ in possible_classes:
-                reduced_data_model["properties"].extend(properties)
-
-        try:
-            return type(self.rules)(**reduced_data_model)
-        except ValidationError as e:
-            warnings.warn(f"Reduced data model is not complete: {e}", stacklevel=2)
-            reduced_data_model["metadata"].schema_ = SchemaCompleteness.partial
-            return type(self.rules).model_construct(**reduced_data_model)
-
-
-class InformationArchitectRulesAnalysis(_SharedAnalysis[InformationRules, InformationProperty, InformationClass]):
-    """Assumes analysis over only the complete schema"""
-
-    def has_hop_transformations(self):
-        return any(
-            prop_.transformation and isinstance(prop_.transformation.traversal, Hop) for prop_ in self.rules.properties
-        )
-
-    def define_property_renaming_config(self) -> dict[str, str]:
-        # placeholder comes in new PR
-        return {}
-
-
-class AssetArchitectRulesAnalysis(_SharedAnalysis[AssetRules, AssetProperty, AssetClass]):
-    """Assumes analysis over only the complete schema"""
-
-    def class_property_pairs(
-        self,
-        only_rdfpath: bool = False,
-        consider_inheritance: bool = False,
-        implementation_type: EntityTypes = EntityTypes.asset,
-    ) -> dict[ClassEntity, dict[str, AssetProperty]]:
-        class_property_pairs = {}
-
-        T_implementation = AssetEntity if implementation_type == EntityTypes.asset else RelationshipEntity
-
-        for class_, properties in self.classes_with_properties(consider_inheritance).items():
-            processed_properties = {}
-            for property_ in properties:
-                if property_.property_ in processed_properties:
-                    # TODO: use appropriate Warning class from _exceptions.py
-                    # if missing make one !
-                    warnings.warn(
-                        f"Property {property_.property_} for {class_} has been defined more than once!"
-                        " Only the first definition will be considered, skipping the rest..",
-                        stacklevel=2,
-                    )
-                    continue
-
-                if (
-                    property_.implementation
-                    and any(isinstance(implementation, T_implementation) for implementation in property_.implementation)
-                    and (not only_rdfpath or (only_rdfpath and isinstance(property_.transformation, RDFPath)))
-                ):
-                    implementation = [
-                        implementation
-                        for implementation in property_.implementation
-                        if isinstance(implementation, T_implementation)
-                    ]
-
-                    processed_properties[property_.property_] = property_.model_copy(
-                        deep=True, update={"implementation": implementation}
-                    )
-
-            if processed_properties:
-                class_property_pairs[class_] = processed_properties
-
-        return class_property_pairs
-
-    def asset_definition(
-        self, only_rdfpath: bool = False, consider_inheritance: bool = False
-    ) -> dict[ClassEntity, dict[str, AssetProperty]]:
-        return self.class_property_pairs(
-            consider_inheritance=consider_inheritance,
-            only_rdfpath=only_rdfpath,
-            implementation_type=EntityTypes.asset,
-        )
-
-    def relationship_definition(
-        self, only_rdfpath: bool = False, consider_inheritance: bool = False
-    ) -> dict[ClassEntity, dict[str, AssetProperty]]:
-        return self.class_property_pairs(
-            consider_inheritance=consider_inheritance,
-            only_rdfpath=only_rdfpath,
-            implementation_type=EntityTypes.relationship,
-        )

cognite/neat/utils/cdf.py

@@ -1,59 +0,0 @@
-from cognite.client import CogniteClient
-from cognite.client.data_classes import filters
-from pydantic import BaseModel, field_validator
-
-
-class CogniteClientConfig(BaseModel):
-    project: str = "dev"
-    client_id: str = "neat"
-    base_url: str = "https://api.cognitedata.com"
-    scopes: list[str] = ["project:read", "project:write"]
-    timeout: int = 60
-    max_workers: int = 3
-
-    @field_validator("scopes", mode="before")
-    def string_to_list(cls, value):
-        return [value] if isinstance(value, str) else value
-
-
-class InteractiveCogniteClient(CogniteClientConfig):
-    authority_url: str
-    redirect_port: int = 53_000
-
-
-class ServiceCogniteClient(CogniteClientConfig):
-    token_url: str = "https://login.microsoftonline.com/common/oauth2/token"
-    client_secret: str = "secret"
-
-
-def clean_space(client: CogniteClient, space: str) -> None:
-    """Deletes all data in a space.
-
-    This means all nodes, edges, views, containers, and data models located in the given space.
-
-    Args:
-        client: Connected CogniteClient
-        space: The space to delete.
-
-    """
-    edges = client.data_modeling.instances.list("edge", limit=-1, filter=filters.Equals(["edge", "space"], space))
-    if edges:
-        instances = client.data_modeling.instances.delete(edges=edges.as_ids())
-        print(f"Deleted {len(instances.edges)} edges")
-    nodes = client.data_modeling.instances.list("node", limit=-1, filter=filters.Equals(["node", "space"], space))
-    if nodes:
-        instances = client.data_modeling.instances.delete(nodes=nodes.as_ids())
-        print(f"Deleted {len(instances.nodes)} nodes")
-    views = client.data_modeling.views.list(limit=-1, space=space)
-    if views:
-        deleted_views = client.data_modeling.views.delete(views.as_ids())
-        print(f"Deleted {len(deleted_views)} views")
-    containers = client.data_modeling.containers.list(limit=-1, space=space)
-    if containers:
-        deleted_containers = client.data_modeling.containers.delete(containers.as_ids())
-        print(f"Deleted {len(deleted_containers)} containers")
-    if data_models := client.data_modeling.data_models.list(limit=-1, space=space):
-        deleted_data_models = client.data_modeling.data_models.delete(data_models.as_ids())
-        print(f"Deleted {len(deleted_data_models)} data models")
-    deleted_space = client.data_modeling.spaces.delete(space)
-    print(f"Deleted space {deleted_space}")

cognite/neat/utils/cdf_loaders/data_classes.py

@@ -1,121 +0,0 @@
-from abc import ABC
-from dataclasses import dataclass
-from typing import Any, cast
-
-from cognite.client import CogniteClient
-from cognite.client.data_classes._base import (
-    CogniteResourceList,
-    WriteableCogniteResource,
-    WriteableCogniteResourceList,
-)
-
-# The Table, TableWrite data classes in the Cognite-SDK lacks the database attribute.
-# This is a problem when creating the RawTableLoader that needs the data class to be able to create, update, retrieve
-# and delete tables.
-# This is a reimplemented version of the Table, TableWrite data classes with the database attribute added.
-
-
-@dataclass(frozen=True)
-class RawTableID:
-    table: str
-    database: str
-
-    def as_tuple(self) -> tuple[str, str]:
-        return self.database, self.table
-
-
-class RawTableCore(WriteableCogniteResource["RawTableWrite"], ABC):
-    """A NoSQL database table to store customer data
-
-    Args:
-        name (str | None): Unique name of the table
-    """
-
-    def __init__(
-        self,
-        name: str | None = None,
-        database: str | None = None,
-    ) -> None:
-        self.name = name
-        self.database = database
-
-    def as_id(self) -> RawTableID:
-        if self.name is None or self.database is None:
-            raise ValueError("name and database are required to create a TableID")
-        return RawTableID(table=self.name, database=self.database)
-
-
-class RawTable(RawTableCore):
-    """A NoSQL database table to store customer data.
-    This is the reading version of the Table class, which is used when retrieving a table.
-
-    Args:
-        name (str | None): Unique name of the table
-        created_time (int | None): Time the table was created.
-        cognite_client (CogniteClient | None): The client to associate with this object.
-    """
-
-    def __init__(
-        self,
-        name: str | None = None,
-        database: str | None = None,
-        created_time: int | None = None,
-        cognite_client: CogniteClient | None = None,
-    ) -> None:
-        super().__init__(name, database)
-        self.created_time = created_time
-        self._cognite_client = cast("CogniteClient", cognite_client)
-
-        self._db_name: str | None = None
-
-    def as_write(self) -> "RawTableWrite":
-        """Returns this Table as a TableWrite"""
-        if self.name is None or self.database is None:
-            raise ValueError("name and database are required to create a Table")
-        return RawTableWrite(name=self.name, database=self.database)
-
-
-class RawTableWrite(RawTableCore):
-    """A NoSQL database table to store customer data
-    This is the writing version of the Table class, which is used when creating a table.
-
-    Args:
-        name (str): Unique name of the table
-    """
-
-    def __init__(
-        self,
-        name: str,
-        database: str,
-    ) -> None:
-        super().__init__(name, database)
-
-    @classmethod
-    def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> "RawTableWrite":
-        return cls(resource["name"], resource["database"])
-
-    def as_write(self) -> "RawTableWrite":
-        """Returns this TableWrite instance."""
-        return self
-
-
-class RawTableWriteList(CogniteResourceList[RawTableWrite]):
-    _RESOURCE = RawTableWrite
-
-    def as_ids(self) -> list[RawTableID]:
-        """Returns this TableWriteList as a list of TableIDs"""
-        return [table.as_id() for table in self.data]
-
-
-class RawTableList(
-    WriteableCogniteResourceList[RawTableWrite, RawTable],
-):
-    _RESOURCE = RawTable
-
-    def as_write(self) -> RawTableWriteList:
-        """Returns this TableList as a TableWriteList"""
-        return RawTableWriteList([table.as_write() for table in self.data])
-
-    def as_ids(self) -> list[RawTableID]:
-        """Returns this TableList as a list of TableIDs"""
-        return [table.as_id() for table in self.data]