cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl

cognite/neat/_rules/models/asset/_rules.py

@@ -1,109 +0,0 @@
-import sys
-from typing import Any, ClassVar, cast
-
-from pydantic import Field, field_validator, model_validator
-from rdflib import Namespace, URIRef
-
-from cognite.neat._constants import DEFAULT_NAMESPACE, get_default_prefixes
-from cognite.neat._rules.models._base_rules import BaseRules, RoleTypes, SheetList
-from cognite.neat._rules.models.entities import (
-    CdfResourceEntityList,
-    ClassEntity,
-    MultiValueTypeInfo,
-    Undefined,
-)
-from cognite.neat._rules.models.information import (
-    InformationClass,
-    InformationMetadata,
-    InformationProperty,
-    InformationRules,
-)
-
-if sys.version_info >= (3, 11):
-    from typing import Self
-else:
-    from typing_extensions import Self
-
-
-class AssetMetadata(InformationMetadata):
-    role: ClassVar[RoleTypes] = RoleTypes.asset
-
-
-class AssetClass(InformationClass): ...
-
-
-class AssetProperty(InformationProperty):
-    """
-    A property is a characteristic of a class. It is a named attribute of a class that describes a range of values
-    or a relationship to another class.
-
-    Args:
-        class_: Class ID to which property belongs
-        property_: Property ID of the property
-        name: Property name.
-        value_type: Type of value property will hold (data or link to another class)
-        min_count: Minimum count of the property values. Defaults to 0
-        max_count: Maximum count of the property values. Defaults to None
-        default: Default value of the property
-        reference: Reference to the source of the information, HTTP URI
-        match_type: The match type of the resource being described and the source entity.
-        transformation: Actual rule for the transformation from source to target representation of
-              knowledge graph. Defaults to None (no transformation)
-        implementation: Details on how given class-property is implemented in the classic CDF
-    """
-
-    implementation: CdfResourceEntityList = Field(alias="Implementation")
-
-
-class AssetRules(BaseRules):
-    metadata: AssetMetadata = Field(alias="Metadata")
-    properties: SheetList[AssetProperty] = Field(alias="Properties")
-    classes: SheetList[AssetClass] = Field(alias="Classes")
-    prefixes: dict[str, Namespace] = Field(default_factory=get_default_prefixes)
-    last: "AssetRules | None" = Field(None, alias="Last")
-    reference: "AssetRules | None" = Field(None, alias="Reference")
-
-    @field_validator("prefixes", mode="before")
-    def parse_str(cls, values: Any) -> Any:
-        if isinstance(values, dict):
-            return {key: Namespace(value) if isinstance(value, str) else value for key, value in values.items()}
-        return values
-
-    @model_validator(mode="after")
-    def update_entities_prefix(self) -> Self:
-        # update expected_value_types
-        for property_ in self.properties:
-            if isinstance(property_.value_type, ClassEntity) and property_.value_type.prefix is Undefined:
-                property_.value_type.prefix = self.metadata.prefix
-
-            if isinstance(property_.value_type, MultiValueTypeInfo):
-                property_.value_type.set_default_prefix(self.metadata.prefix)
-
-            if property_.class_.prefix is Undefined:
-                property_.class_.prefix = self.metadata.prefix
-
-        # update parent classes
-        for class_ in self.classes:
-            if class_.parent:
-                for parent in class_.parent:
-                    if not isinstance(parent.prefix, str):
-                        parent.prefix = self.metadata.prefix
-            if class_.class_.prefix is Undefined:
-                class_.class_.prefix = self.metadata.prefix
-
-        return self
-
-    @model_validator(mode="after")
-    def post_validation(self) -> "AssetRules":
-        from ._validation import AssetPostValidation
-
-        issue_list = AssetPostValidation(cast(InformationRules, self)).validate()
-        if issue_list.warnings:
-            issue_list.trigger_warnings()
-        if issue_list.has_errors:
-            raise issue_list.as_exception()
-        return self
-
-    @property
-    def id_(self) -> URIRef:
-        return DEFAULT_NAMESPACE[f"data-model/verified/asset/{self.metadata.prefix}/{self.metadata.version}"]

cognite/neat/_rules/models/asset/_rules_input.py

@@ -1,101 +0,0 @@
-from dataclasses import dataclass
-from typing import Any
-
-from rdflib import Namespace, URIRef
-
-from cognite.neat._constants import DEFAULT_NAMESPACE
-from cognite.neat._rules.models._base_input import InputComponent, InputRules
-from cognite.neat._rules.models.data_types import DataType
-from cognite.neat._rules.models.entities import (
-    ClassEntity,
-    MultiValueTypeInfo,
-    UnknownEntity,
-    load_value_type,
-)
-from cognite.neat._rules.models.information._rules_input import InformationInputClass, InformationInputMetadata
-
-from ._rules import AssetClass, AssetMetadata, AssetProperty, AssetRules
-
-
-@dataclass
-class AssetInputMetadata(InformationInputMetadata):
-    @classmethod
-    def _get_verified_cls(cls) -> type[AssetMetadata]:
-        return AssetMetadata
-
-
-@dataclass
-class AssetInputProperty(InputComponent[AssetProperty]):
-    class_: ClassEntity | str
-    property_: str
-    value_type: DataType | ClassEntity | MultiValueTypeInfo | UnknownEntity | str
-    name: str | None = None
-    description: str | None = None
-    comment: str | None = None
-    min_count: int | None = None
-    max_count: int | float | None = None
-    default: Any | None = None
-    reference: str | None = None
-    match_type: str | None = None
-    transformation: str | None = None
-    implementation: str | None = None
-    # Only used internally
-    inherited: bool = False
-
-    @classmethod
-    def _get_verified_cls(cls) -> type[AssetProperty]:
-        return AssetProperty
-
-    def dump(self, default_prefix: str) -> dict[str, Any]:  # type: ignore[override]
-        output = super().dump()
-        output["Class"] = ClassEntity.load(self.class_, prefix=default_prefix)
-        output["Value Type"] = load_value_type(self.value_type, default_prefix)
-        return output
-
-
-@dataclass
-class AssetInputClass(InformationInputClass):
-    @classmethod
-    def _get_verified_cls(cls) -> type[AssetClass]:
-        return AssetClass
-
-
-@dataclass
-class AssetInputRules(InputRules[AssetRules]):
-    metadata: AssetInputMetadata
-    properties: list[AssetInputProperty]
-    classes: list[AssetInputClass]
-    prefixes: dict[str, Namespace] | None = None
-    last: "AssetInputRules | None" = None
-    reference: "AssetInputRules | None" = None
-
-    @classmethod
-    def _get_verified_cls(cls) -> type[AssetRules]:
-        return AssetRules
-
-    def dump(self) -> dict[str, Any]:
-        default_prefix = self.metadata.prefix
-        reference: dict[str, Any] | None = None
-        if isinstance(self.reference, AssetInputRules):
-            reference = self.reference.dump()
-        elif isinstance(self.reference, AssetRules):
-            # We need to load through the AssetRulesInput to set the correct default space and version
-            reference = AssetInputRules.load(self.reference.model_dump()).dump()
-        last: dict[str, Any] | None = None
-        if isinstance(self.last, AssetInputRules):
-            last = self.last.dump()
-        elif isinstance(self.last, AssetRules):
-            # We need to load through the AssetRulesInput to set the correct default space and version
-            last = AssetInputRules.load(self.last.model_dump()).dump()
-
-        return dict(
-            Metadata=self.metadata.dump(),
-            Properties=[prop.dump(default_prefix) for prop in self.properties],
-            Classes=[class_.dump(default_prefix) for class_ in self.classes],
-            Last=last,
-            Reference=reference,
-        )
-
-    @property
-    def id_(self) -> URIRef:
-        return DEFAULT_NAMESPACE[f"data-model/unverified/asset/{self.metadata.prefix}/{self.metadata.version}"]

cognite/neat/_rules/models/asset/_validation.py

@@ -1,45 +0,0 @@
-from graphlib import CycleError
-from typing import cast
-
-from cognite.neat._issues import IssueList
-from cognite.neat._issues.errors import NeatValueError, PropertyDefinitionError
-from cognite.neat._rules.models._base_rules import SheetList
-from cognite.neat._rules.models.asset._rules import AssetProperty, AssetRules
-from cognite.neat._rules.models.entities import AssetEntity, AssetFields, ClassEntity
-from cognite.neat._rules.models.information._validation import InformationPostValidation
-
-
-class AssetPostValidation(InformationPostValidation):
-    def validate(self) -> IssueList:
-        self.issue_list = super().validate()
-        self._parent_property_point_to_class()
-        self._circular_dependency()
-        return self.issue_list
-
-    def _parent_property_point_to_class(self) -> None:
-        for property_ in cast(SheetList[AssetProperty], self.properties):
-            for implementation in property_.implementation:
-                if (
-                    isinstance(implementation, AssetEntity)
-                    and implementation.property_ == AssetFields.parentExternalId
-                    and not isinstance(property_.value_type, ClassEntity)
-                ):
-                    self.issue_list.append(
-                        PropertyDefinitionError(
-                            property_.class_,
-                            "class",
-                            property_.property_,
-                            "parentExternalId is only allowed to "
-                            f"point to a Class not {type(property_.value_type).__name__}",
-                        )
-                    )
-
-    def _circular_dependency(self) -> None:
-        from cognite.neat._rules.analysis import AssetAnalysis
-
-        try:
-            _ = AssetAnalysis(cast(AssetRules, self.rules)).class_topological_sort()
-        except CycleError as error:
-            self.issue_list.append(
-                NeatValueError(f"Invalid Asset Hierarchy, circular dependency detected: {error.args[1]}")
-            )

cognite/neat/_rules/models/domain.py

@@ -1,136 +0,0 @@
-import math
-from collections.abc import Hashable
-from dataclasses import dataclass, field
-from typing import ClassVar
-
-from pydantic import Field, field_serializer, field_validator
-from rdflib import URIRef
-
-from cognite.neat._constants import DEFAULT_NAMESPACE
-from cognite.neat._rules.models.data_types import DataType
-from cognite.neat._rules.models.entities import ClassEntity, ClassEntityList
-
-from ._base_input import InputComponent, InputRules
-from ._base_rules import (
-    BaseMetadata,
-    BaseRules,
-    RoleTypes,
-    SheetList,
-    SheetRow,
-)
-from ._types import ClassEntityType, InformationPropertyType, StrOrListType
-
-
-class DomainMetadata(BaseMetadata):
-    role: ClassVar[RoleTypes] = RoleTypes.domain_expert
-    creator: StrOrListType
-
-    def as_identifier(self) -> str:
-        return "DomainRules"
-
-    def get_prefix(self) -> str:
-        return "domain"
-
-
-class DomainProperty(SheetRow):
-    class_: ClassEntityType = Field(alias="Class")
-    property_: InformationPropertyType = Field(alias="Property")
-    name: str | None = Field(alias="Name", default=None)
-    description: str | None = Field(alias="Description", default=None)
-    value_type: DataType | ClassEntity = Field(alias="Value Type")
-    min_count: int | None = Field(alias="Min Count", default=None)
-    max_count: int | float | None = Field(alias="Max Count", default=None)
-
-    def _identifier(self) -> tuple[Hashable, ...]:
-        return self.class_, self.property_
-
-    @field_serializer("max_count", when_used="json-unless-none")
-    def serialize_max_count(self, value: int | float | None) -> int | float | None | str:
-        if isinstance(value, float) and math.isinf(value):
-            return None
-        return value
-
-    @field_validator("max_count", mode="before")
-    def parse_max_count(cls, value: int | float | None) -> int | float | None:
-        if value is None:
-            return float("inf")
-        return value
-
-
-class DomainClass(SheetRow):
-    class_: ClassEntityType = Field(alias="Class")
-    name: str | None = Field(alias="Name", default=None)
-    description: str | None = Field(None, alias="Description")
-    parent: ClassEntityList | None = Field(alias="Parent Class")
-
-    def _identifier(self) -> tuple[Hashable, ...]:
-        return (self.class_,)
-
-    @field_serializer("parent", when_used="unless-none")
-    def serialize_parent(self, value: list[ClassEntity]) -> str:
-        return ",".join([str(entry) for entry in value])
-
-
-class DomainRules(BaseRules):
-    metadata: DomainMetadata = Field(alias="Metadata")
-    properties: SheetList[DomainProperty] = Field(alias="Properties")
-    classes: SheetList[DomainClass] | None = Field(None, alias="Classes")
-    last: "DomainRules | None" = Field(None, alias="Last")
-    reference: "DomainRules | None" = Field(None, alias="Reference")
-
-    @property
-    def id_(self) -> URIRef:
-        return DEFAULT_NAMESPACE["data-model/verified/domain"]
-
-
-@dataclass
-class DomainInputMetadata(InputComponent[DomainMetadata]):
-    creator: str
-
-    @classmethod
-    def _get_verified_cls(cls) -> type[DomainMetadata]:
-        return DomainMetadata
-
-
-@dataclass
-class DomainInputProperty(InputComponent[DomainProperty]):
-    class_: str
-    property_: str
-    value_type: str
-    name: str | None = None
-    description: str | None = None
-    min_count: int | None = None
-    max_count: int | float | None = None
-
-    @classmethod
-    def _get_verified_cls(cls) -> type[DomainProperty]:
-        return DomainProperty
-
-
-@dataclass
-class DomainInputClass(InputComponent[DomainClass]):
-    class_: str
-    name: str | None = None
-    description: str | None = None
-    parent: list[str] | None = None
-
-    @classmethod
-    def _get_verified_cls(cls) -> type[DomainClass]:
-        return DomainClass
-
-
-@dataclass
-class DomainInputRules(InputRules[DomainRules]):
-    metadata: DomainInputMetadata
-    properties: list[DomainInputProperty] = field(default_factory=list)
-    classes: list[DomainInputClass] = field(default_factory=list)
-    last: "DomainInputRules | None" = None
-    reference: "DomainInputRules | None" = None
-
-    @classmethod
-    def _get_verified_cls(cls) -> type[DomainRules]:
-        return DomainRules
-
-    @property
-    def id_(self) -> URIRef:
-        return DEFAULT_NAMESPACE["data-model/unverified/domain"]

cognite/neat/_rules/models/mapping/_base.py

@@ -1,131 +0,0 @@
-from collections import Counter, defaultdict
-from collections.abc import Iterator, MutableSequence, Sequence
-from pathlib import Path
-from typing import Any, Generic, SupportsIndex, TypeVar, cast, get_args, overload
-
-import pandas as pd
-from pydantic import BaseModel, GetCoreSchemaHandler, field_validator
-from pydantic_core import core_schema
-from pydantic_core.core_schema import ValidationInfo
-
-from cognite.neat._issues.errors import NeatValueError
-from cognite.neat._rules.models._base_rules import ClassRef, PropertyRef
-from cognite.neat._rules.models.entities import ClassEntity, Undefined
-
-T_Mapping = TypeVar("T_Mapping", bound=ClassRef | PropertyRef)
-
-
-class Mapping(BaseModel, Generic[T_Mapping]):
-    source: T_Mapping
-    destination: T_Mapping
-
-
-class MappingList(list, MutableSequence[Mapping[T_Mapping]]):
-    @classmethod
-    def __get_pydantic_core_schema__(cls, source: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema:
-        if args := get_args(source):
-            item_type = args[0]
-        else:
-            # Someone use SheetList without specifying the type
-            raise TypeError("SheetList must be used with a type argument, e.g., SheetList[InformationProperty]")
-
-        instance_schema = core_schema.is_instance_schema(cls)
-        sequence_row_schema = handler.generate_schema(Sequence[item_type])  # type: ignore[valid-type]
-
-        non_instance_schema = core_schema.no_info_after_validator_function(MappingList, sequence_row_schema)
-        return core_schema.union_schema([instance_schema, non_instance_schema])
-
-    def as_destination_by_source(self) -> dict[T_Mapping, T_Mapping]:
-        return {mapping.source: mapping.destination for mapping in self}
-
-    def to_pandas(self, drop_na_columns: bool = True, include: list[str] | None = None) -> pd.DataFrame:
-        """Converts ResourceDict to pandas DataFrame."""
-        df = pd.DataFrame([entity.model_dump() for entity in self])
-        if drop_na_columns:
-            df = df.dropna(axis=1, how="all")
-        if include is not None:
-            df = df[include]
-        return df
-
-    def _repr_html_(self) -> str:
-        """Returns HTML representation of ResourceDict."""
-        return self.to_pandas(drop_na_columns=True)._repr_html_()  # type: ignore[operator]
-
-    # Implemented to get correct type hints
-    def __iter__(self) -> Iterator[Mapping[T_Mapping]]:
-        return super().__iter__()
-
-    @overload
-    def __getitem__(self, index: SupportsIndex) -> Mapping[T_Mapping]: ...
-
-    @overload
-    def __getitem__(self, index: slice) -> "MappingList[T_Mapping]": ...
-
-    def __getitem__(self, index: SupportsIndex | slice, /) -> "Mapping[T_Mapping] | MappingList[T_Mapping]":
-        if isinstance(index, slice):
-            return MappingList[T_Mapping](super().__getitem__(index))
-        return super().__getitem__(index)
-
-
-class RuleMapping(BaseModel):
-    properties: MappingList[PropertyRef]
-    classes: MappingList[ClassRef]
-
-    @field_validator("properties", "classes", mode="before")
-    def as_mapping_list(cls, value: Sequence[Any], info: ValidationInfo) -> Any:
-        if isinstance(value, Sequence) and not isinstance(value, MappingList):
-            annotation = cast(type, cls.model_fields[info.field_name].annotation)  # type: ignore[index]
-            ref_cls = get_args(annotation)[0]
-            return annotation([Mapping[ref_cls].model_validate(item) for item in value])  # type: ignore[valid-type, index]
-        return value
-
-    @classmethod
-    def load_spreadsheet(
-        cls, path: str | Path, source_prefix: str | None = None, destination_prefix: str | None = None
-    ) -> "RuleMapping":
-        """Loads mapping from Excel spreadsheet.
-
-        This method expects four columns in the spreadsheet. The first two columns are the source class and
-        property, and the last two columns are the destination class and property. The method will create
-        a mapping for each row in the spreadsheet.
-
-        The class mapping will be inferred from the property mappings. If a source class has multiple
-        destination classes, the most common destination class will be used.
-
-        Args:
-            path: Path to Excel spreadsheet.
-            source_prefix: Default prefix for source classes.
-            destination_prefix: Default prefix for destination classes.
-
-        Returns:
-            Mapping object.
-
-        """
-        df = pd.read_excel(path).dropna(axis=1, how="all")
-        properties = MappingList[PropertyRef]()
-        destination_classes_by_source: dict[ClassEntity, Counter[ClassEntity]] = defaultdict(Counter)
-        for _, row in df.iterrows():
-            if len(row) < 4:
-                raise NeatValueError(f"Row {row} is not valid. Expected 4 columns, got {len(row)}")
-
-            if any(pd.isna(row.iloc[:4])):
-                continue
-            source_class, source_property, destination_class, destination_property = row.iloc[:4]
-            source_entity = ClassEntity.load(source_class, prefix=source_prefix or Undefined)
-            destination_entity = ClassEntity.load(destination_class, prefix=destination_prefix or Undefined)
-            properties.append(
-                Mapping(
-                    source=PropertyRef(Class=source_entity, Property=source_property),
-                    destination=PropertyRef(Class=destination_entity, Property=destination_property),
-                )
-            )
-            destination_classes_by_source[source_entity][destination_entity] += 1
-
-        classes = MappingList[ClassRef]()
-        for source_entity, destination_classes in destination_classes_by_source.items():
-            destination_entity = destination_classes.most_common(1)[0][0]
-            classes.append(
-                Mapping(source=ClassRef(Class=source_entity), destination=ClassRef(Class=destination_entity))
-            )
-
-        return cls(properties=properties, classes=classes)

cognite/neat/_utils/cdf/loaders/__init__.py

@@ -1,25 +0,0 @@
-from ._base import ResourceLoader
-from ._data_modeling import (
-    ContainerLoader,
-    DataModelingLoader,
-    DataModelLoader,
-    SpaceLoader,
-    ViewLoader,
-)
-from ._ingestion import (
-    RawDatabaseLoader,
-    RawTableLoader,
-    TransformationLoader,
-)
-
-__all__ = [
-    "DataModelingLoader",
-    "ContainerLoader",
-    "DataModelLoader",
-    "ResourceLoader",
-    "SpaceLoader",
-    "ViewLoader",
-    "TransformationLoader",
-    "RawTableLoader",
-    "RawDatabaseLoader",
-]

cognite/neat/_utils/cdf/loaders/_base.py

@@ -1,54 +0,0 @@
-from abc import ABC, abstractmethod
-from collections.abc import Sequence
-from typing import Generic, TypeVar
-
-from cognite.client import CogniteClient
-from cognite.client.data_classes._base import (
-    T_CogniteResourceList,
-    T_WritableCogniteResource,
-    T_WriteClass,
-    WriteableCogniteResourceList,
-)
-from cognite.client.utils.useful_types import SequenceNotStr
-
-from cognite.neat._shared import T_ID
-
-T_WritableCogniteResourceList = TypeVar("T_WritableCogniteResourceList", bound=WriteableCogniteResourceList)
-
-
-class ResourceLoader(
-    ABC,
-    Generic[T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList],
-):
-    resource_name: str
-
-    def __init__(self, client: CogniteClient) -> None:
-        self.client = client
-
-    @classmethod
-    @abstractmethod
-    def get_id(cls, item: T_WriteClass | T_WritableCogniteResource | dict | T_ID) -> T_ID:
-        raise NotImplementedError
-
-    @classmethod
-    def get_ids(cls, items: Sequence[T_WriteClass | T_WritableCogniteResource]) -> list[T_ID]:
-        return [cls.get_id(item) for item in items]
-
-    @abstractmethod
-    def create(self, items: Sequence[T_WriteClass]) -> T_WritableCogniteResourceList:
-        raise NotImplementedError
-
-    @abstractmethod
-    def retrieve(self, ids: SequenceNotStr[T_ID]) -> T_WritableCogniteResourceList:
-        raise NotImplementedError
-
-    @abstractmethod
-    def update(self, items: Sequence[T_WriteClass]) -> T_WritableCogniteResourceList:
-        raise NotImplementedError
-
-    @abstractmethod
-    def delete(self, ids: SequenceNotStr[T_ID]) -> list[T_ID]:
-        raise NotImplementedError
-
-    def are_equal(self, local: T_WriteClass, remote: T_WritableCogniteResource) -> bool:
-        return local == remote.as_write()