cognite-neat 0.88.2__py3-none-any.whl → 0.89.0__py3-none-any.whl

cognite/neat/rules/models/information/_rules_input.py

@@ -1,22 +1,16 @@
-from collections.abc import Sequence
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
-from typing import Any, Literal, cast, overload
+from typing import Any, Literal
 
 from rdflib import Namespace
 
-from cognite.neat.rules.models._base import (
-    DataModelType,
-    ExtensionCategory,
-    SchemaCompleteness,
-    _add_alias,
-)
+from cognite.neat.rules.models._base_input import InputComponent, InputRules
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import (
     ClassEntity,
     MultiValueTypeInfo,
-    Unknown,
     UnknownEntity,
+    load_value_type,
 )
 
 from ._rules import (
@@ -28,7 +22,7 @@ from ._rules import (
 
 
 @dataclass
-class InformationMetadataInput:
+class InformationInputMetadata(InputComponent[InformationMetadata]):
     schema_: Literal["complete", "partial", "extended"]
     prefix: str
     namespace: str
@@ -44,47 +38,23 @@ class InformationMetadataInput:
     rights: str | None = None
 
     @classmethod
-    def load(cls, data: dict[str, Any] | None) -> "InformationMetadataInput | None":
-        if data is None:
-            return None
-        _add_alias(data, InformationMetadata)
-        return cls(
-            data_model_type=data.get("data_model_type", "enterprise"),
-            extension=data.get("extension", "addition"),
-            schema_=data.get("schema_", "partial"),  # type: ignore[arg-type]
-            version=data.get("version"),  # type: ignore[arg-type]
-            namespace=data.get("namespace"),  # type: ignore[arg-type]
-            prefix=data.get("prefix"),  # type: ignore[arg-type]
-            name=data.get("name"),
-            creator=data.get("creator"),  # type: ignore[arg-type]
-            description=data.get("description"),
-            created=data.get("created"),
-            updated=data.get("updated"),
-            license=data.get("license"),
-            rights=data.get("rights"),
-        )
+    def _get_verified_cls(cls) -> type[InformationMetadata]:
+        return InformationMetadata
 
-    def dump(self) -> dict[str, Any]:
-        return dict(
-            dataModelType=DataModelType(self.data_model_type),
-            schema=SchemaCompleteness(self.schema_),
-            extension=ExtensionCategory(self.extension),
-            namespace=Namespace(self.namespace),
-            prefix=self.prefix,
-            version=self.version,
-            name=self.name,
-            creator=self.creator,
-            description=self.description,
-            created=self.created or datetime.now(),
-            updated=self.updated or datetime.now(),
-        )
+    def dump(self, **kwargs) -> dict[str, Any]:
+        output = super().dump()
+        if self.created is None:
+            output["created"] = datetime.now()
+        if self.updated is None:
+            output["updated"] = datetime.now()
+        return output
 
 
 @dataclass
-class InformationPropertyInput:
-    class_: str
+class InformationInputProperty(InputComponent[InformationProperty]):
+    class_: ClassEntity | str
     property_: str
-    value_type: str
+    value_type: DataType | ClassEntity | MultiValueTypeInfo | UnknownEntity | str
     name: str | None = None
     description: str | None = None
     comment: str | None = None
@@ -94,194 +64,77 @@ class InformationPropertyInput:
     reference: str | None = None
     match_type: str | None = None
     transformation: str | None = None
+    # Only used internally
+    inherited: bool = False
 
     @classmethod
-    @overload
-    def load(cls, data: None) -> None: ...
-
-    @classmethod
-    @overload
-    def load(cls, data: dict[str, Any]) -> "InformationPropertyInput": ...
-
-    @classmethod
-    @overload
-    def load(cls, data: list[dict[str, Any]]) -> list["InformationPropertyInput"]: ...
-
-    @classmethod
-    def load(
-        cls, data: dict[str, Any] | list[dict[str, Any]] | None
-    ) -> "InformationPropertyInput | list[InformationPropertyInput] | None":
-        if data is None:
-            return None
-        if isinstance(data, list) or (isinstance(data, dict) and isinstance(data.get("data"), list)):
-            items = cast(
-                list[dict[str, Any]],
-                data.get("data") if isinstance(data, dict) else data,
-            )
-            return [loaded for item in items if (loaded := cls.load(item)) is not None]
-
-        _add_alias(data, InformationProperty)
-        return cls(
-            class_=data.get("class_"),  # type: ignore[arg-type]
-            property_=data.get("property_"),  # type: ignore[arg-type]
-            name=data.get("name", None),
-            description=data.get("description", None),
-            comment=data.get("comment", None),
-            value_type=data.get("value_type"),  # type: ignore[arg-type]
-            min_count=data.get("min_count", None),
-            max_count=data.get("max_count", None),
-            default=data.get("default", None),
-            reference=data.get("reference", None),
-            match_type=data.get("match_type", None),
-            transformation=data.get("transformation", None),
-        )
-
-    def dump(self, default_prefix: str) -> dict[str, Any]:
-        value_type: MultiValueTypeInfo | DataType | ClassEntity | UnknownEntity
-
-        # property holding xsd data type
-        # check if it is multi value type
-        if "|" in self.value_type:
-            value_type = MultiValueTypeInfo.load(self.value_type)
-            value_type.set_default_prefix(default_prefix)
-
-        elif DataType.is_data_type(self.value_type):
-            value_type = DataType.load(self.value_type)
+    def _get_verified_cls(cls) -> type[InformationProperty]:
+        return InformationProperty
 
-        # unknown value type
-        elif self.value_type == str(Unknown):
-            value_type = UnknownEntity()
-
-        # property holding link to class
-        else:
-            value_type = ClassEntity.load(self.value_type, prefix=default_prefix)
-
-        return {
-            "Class": ClassEntity.load(self.class_, prefix=default_prefix),
-            "Property": self.property_,
-            "Name": self.name,
-            "Description": self.description,
-            "Comment": self.comment,
-            "Value Type": value_type,
-            "Min Count": self.min_count,
-            "Max Count": self.max_count,
-            "Default": self.default,
-            "Reference": self.reference,
-            "Match Type": self.match_type,
-            "Transformation": self.transformation,
-        }
+    def dump(self, default_prefix: str, **kwargs) -> dict[str, Any]:  # type: ignore[override]
+        output = super().dump()
+        output["Class"] = ClassEntity.load(self.class_, prefix=default_prefix)
+        output["Value Type"] = load_value_type(self.value_type, default_prefix)
+        return output
 
 
 @dataclass
-class InformationClassInput:
-    class_: str
+class InformationInputClass(InputComponent[InformationClass]):
+    class_: ClassEntity | str
     name: str | None = None
     description: str | None = None
     comment: str | None = None
-    parent: str | None = None
+    parent: str | list[ClassEntity] | None = None
     reference: str | None = None
     match_type: str | None = None
 
     @classmethod
-    @overload
-    def load(cls, data: None) -> None: ...
-
-    @classmethod
-    @overload
-    def load(cls, data: dict[str, Any]) -> "InformationClassInput": ...
+    def _get_verified_cls(cls) -> type[InformationClass]:
+        return InformationClass
 
-    @classmethod
-    @overload
-    def load(cls, data: list[dict[str, Any]]) -> list["InformationClassInput"]: ...
-
-    @classmethod
-    def load(
-        cls, data: dict[str, Any] | list[dict[str, Any]] | None
-    ) -> "InformationClassInput | list[InformationClassInput] | None":
-        if data is None:
-            return None
-        if isinstance(data, list) or (isinstance(data, dict) and isinstance(data.get("data"), list)):
-            items = cast(
-                list[dict[str, Any]],
-                data.get("data") if isinstance(data, dict) else data,
-            )
-            return [loaded for item in items if (loaded := cls.load(item)) is not None]
-        _add_alias(data, InformationClass)
-        return cls(
-            class_=data.get("class_"),  # type: ignore[arg-type]
-            name=data.get("name", None),
-            description=data.get("description", None),
-            comment=data.get("comment", None),
-            parent=data.get("parent", None),
-            reference=data.get("reference", None),
-            match_type=data.get("match_type", None),
-        )
+    @property
+    def class_str(self) -> str:
+        return str(self.class_)
 
-    def dump(self, default_prefix: str) -> dict[str, Any]:
-        return {
-            "Class": ClassEntity.load(self.class_, prefix=default_prefix),
-            "Name": self.name,
-            "Description": self.description,
-            "Comment": self.comment,
-            "Reference": self.reference,
-            "Match Type": self.match_type,
-            "Parent Class": (
-                [ClassEntity.load(parent, prefix=default_prefix) for parent in self.parent.split(",")]
-                if self.parent
-                else None
-            ),
-        }
+    def dump(self, default_prefix: str, **kwargs) -> dict[str, Any]:  # type: ignore[override]
+        output = super().dump()
+        parent: list[ClassEntity] | None = None
+        if isinstance(self.parent, str):
+            parent = [ClassEntity.load(parent, prefix=default_prefix) for parent in self.parent.split(",")]
+        elif isinstance(self.parent, list):
+            parent = [ClassEntity.load(parent_, prefix=default_prefix) for parent_ in self.parent]
+        output["Class"] = ClassEntity.load(self.class_, prefix=default_prefix)
+        output["Parent Class"] = parent
+        return output
 
 
 @dataclass
-class InformationRulesInput:
-    metadata: InformationMetadataInput
-    properties: Sequence[InformationPropertyInput]
-    classes: Sequence[InformationClassInput]
-    prefixes: "dict[str, Namespace] | None" = None
-    last: "InformationRulesInput | InformationRules | None" = None
-    reference: "InformationRulesInput | InformationRules | None" = None
+class InformationInputRules(InputRules[InformationRules]):
+    metadata: InformationInputMetadata
+    properties: list[InformationInputProperty] = field(default_factory=list)
+    classes: list[InformationInputClass] = field(default_factory=list)
+    prefixes: dict[str, Namespace] | None = None
+    last: "InformationInputRules | None" = None
+    reference: "InformationInputRules | None" = None
 
     @classmethod
-    @overload
-    def load(cls, data: dict[str, Any]) -> "InformationRulesInput": ...
-
-    @classmethod
-    @overload
-    def load(cls, data: None) -> None: ...
-
-    @classmethod
-    def load(cls, data: dict | None) -> "InformationRulesInput | None":
-        if data is None:
-            return None
-        _add_alias(data, InformationRules)
-
-        return cls(
-            metadata=InformationMetadataInput.load(data.get("metadata")),  # type: ignore[arg-type]
-            properties=InformationPropertyInput.load(data.get("properties")),  # type: ignore[arg-type]
-            classes=InformationClassInput.load(data.get("classes")),  # type: ignore[arg-type]
-            prefixes=data.get("prefixes"),
-            last=InformationRulesInput.load(data.get("last")),
-            reference=InformationRulesInput.load(data.get("reference")),
-        )
-
-    def as_rules(self) -> InformationRules:
-        return InformationRules.model_validate(self.dump())
+    def _get_verified_cls(cls) -> type[InformationRules]:
+        return InformationRules
 
     def dump(self) -> dict[str, Any]:
         default_prefix = self.metadata.prefix
         reference: dict[str, Any] | None = None
-        if isinstance(self.reference, InformationRulesInput):
+        if isinstance(self.reference, InformationInputRules):
             reference = self.reference.dump()
         elif isinstance(self.reference, InformationRules):
             # We need to load through the InformationRulesInput to set the correct default space and version
-            reference = InformationRulesInput.load(self.reference.model_dump()).dump()
+            reference = InformationInputRules.load(self.reference.model_dump()).dump()
         last: dict[str, Any] | None = None
-        if isinstance(self.last, InformationRulesInput):
+        if isinstance(self.last, InformationInputRules):
             last = self.last.dump()
         elif isinstance(self.last, InformationRules):
             # We need to load through the InformationRulesInput to set the correct default space and version
-            last = InformationRulesInput.load(self.last.model_dump()).dump()
+            last = InformationInputRules.load(self.last.model_dump()).dump()
 
         return dict(
             Metadata=self.metadata.dump(),

cognite/neat/rules/models/information/_validation.py

@@ -3,8 +3,8 @@ from collections import Counter
 from typing import cast
 
 from cognite.neat.issues import IssueList
-from cognite.neat.rules import issues
-from cognite.neat.rules.models._base import DataModelType, SchemaCompleteness
+from cognite.neat.issues.errors import NeatValueError, ResourceNotDefinedError
+from cognite.neat.rules.models._base_rules import DataModelType, SchemaCompleteness
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, UnknownEntity
 from cognite.neat.utils.rdf_ import get_inheritance_path
 
@@ -58,9 +58,9 @@ class InformationPostValidation:
                 ):
                     dangling_classes.add(class_)
 
-        if dangling_classes:
+        for class_ in dangling_classes:
             self.issue_list.append(
-                issues.spreadsheet.ClassNoPropertiesNoParentError([class_.versioned_id for class_ in dangling_classes])
+                NeatValueError(f"Class {class_} has no properties and is not a parent of any class with properties")
             )
 
     def _referenced_parent_classes_exist(self) -> None:
@@ -70,9 +70,15 @@ class InformationPostValidation:
         parents = set(itertools.chain.from_iterable(class_parent_pairs.values()))
 
         if undefined_parents := parents.difference(classes):
-            self.issue_list.append(
-                issues.spreadsheet.ParentClassesNotDefinedError([missing.versioned_id for missing in undefined_parents])
-            )
+            for parent in undefined_parents:
+                # Todo: include row and column number
+                self.issue_list.append(
+                    ResourceNotDefinedError[ClassEntity](
+                        resource_type="class",
+                        identifier=parent,
+                        location="Classes sheet",
+                    )
+                )
 
     def _referenced_classes_exist(self) -> None:
         # needs to be complete for this validation to pass
@@ -83,21 +89,27 @@ class InformationPostValidation:
         if self.metadata.schema_ == SchemaCompleteness.complete and (
             missing_classes := referred_classes.difference(defined_classes)
         ):
-            self.issue_list.append(
-                issues.spreadsheet.PropertiesDefinedForUndefinedClassesError(
-                    [missing.versioned_id for missing in missing_classes]
+            for class_ in missing_classes:
+                self.issue_list.append(
+                    ResourceNotDefinedError[ClassEntity](
+                        resource_type="class",
+                        identifier=class_,
+                        location="Classes sheet",
+                    )
                 )
-            )
 
         # USE CASE: models are extended (user + last = complete)
         if self.metadata.schema_ == SchemaCompleteness.extended:
             defined_classes |= {class_.class_ for class_ in cast(InformationRules, self.rules.last).classes}
             if missing_classes := referred_classes.difference(defined_classes):
-                self.issue_list.append(
-                    issues.spreadsheet.PropertiesDefinedForUndefinedClassesError(
-                        [missing.versioned_id for missing in missing_classes]
+                for class_ in missing_classes:
+                    self.issue_list.append(
+                        ResourceNotDefinedError[ClassEntity](
+                            resource_type="class",
+                            identifier=class_,
+                            location="Classes sheet",
+                        )
                     )
-                )
 
     def _referenced_value_types_exist(self) -> None:
         # adding UnknownEntity to the set of defined classes to handle the case where a property references an unknown
@@ -112,21 +124,29 @@ class InformationPostValidation:
         if self.metadata.schema_ == SchemaCompleteness.complete and (
             missing_value_types := referred_object_types.difference(defined_classes)
         ):
-            self.issue_list.append(
-                issues.spreadsheet.ValueTypeNotDefinedError(
-                    [cast(ClassEntity, missing).versioned_id for missing in missing_value_types]
+            # Todo: include row and column number
+            for missing in missing_value_types:
+                self.issue_list.append(
+                    ResourceNotDefinedError[ClassEntity](
+                        resource_type="class",
+                        identifier=cast(ClassEntity, missing),
+                        location="Classes sheet",
+                    )
                 )
-            )
 
         # USE CASE: models are extended (user + last = complete)
         if self.metadata.schema_ == SchemaCompleteness.extended:
             defined_classes |= {class_.class_ for class_ in cast(InformationRules, self.rules.last).classes}
             if missing_value_types := referred_object_types.difference(defined_classes):
-                self.issue_list.append(
-                    issues.spreadsheet.ValueTypeNotDefinedError(
-                        [cast(ClassEntity, missing).versioned_id for missing in missing_value_types]
+                # Todo: include row and column number
+                for missing in missing_value_types:
+                    self.issue_list.append(
+                        ResourceNotDefinedError(
+                            resource_type="class",
+                            identifier=cast(ClassEntity, missing),
+                            location="Classes sheet",
+                        )
                     )
-                )
 
     def _class_parent_pairs(self) -> dict[ClassEntity, list[ClassEntity]]:
         class_subclass_pairs: dict[ClassEntity, list[ClassEntity]] = {}
@@ -173,7 +193,10 @@ class InformationPostValidation:
             reused_namespaces = [value for value, count in Counter(prefixes.values()).items() if count > 1]
             impacted_prefixes = [key for key, value in prefixes.items() if value in reused_namespaces]
             self.issue_list.append(
-                issues.spreadsheet.PrefixNamespaceCollisionError(
-                    prefixes=impacted_prefixes, namespaces=reused_namespaces
+                NeatValueError(
+                    "Namespace collision detected. The following prefixes "
+                    f"are assigned to the same namespace: {impacted_prefixes}"
+                    f"\nImpacted namespaces: {reused_namespaces}"
+                    "\nMake sure that each unique namespace is assigned to a unique prefix"
                 )
             )

cognite/neat/rules/transformers/__init__.py

@@ -0,0 +1,21 @@
+from ._base import RulesPipeline, RulesTransformer
+from ._converters import AssetToInformation, ConvertToRules, DMSToInformation, InformationToAsset, InformationToDMS
+from ._map_onto import MapOneToOne
+from ._pipelines import ImporterPipeline
+from ._verification import VerifyAnyRules, VerifyAssetRules, VerifyDMSRules, VerifyInformationRules
+
+__all__ = [
+    "ImporterPipeline",
+    "RulesTransformer",
+    "RulesPipeline",
+    "InformationToDMS",
+    "InformationToAsset",
+    "ConvertToRules",
+    "AssetToInformation",
+    "DMSToInformation",
+    "VerifyAssetRules",
+    "VerifyDMSRules",
+    "VerifyInformationRules",
+    "VerifyAnyRules",
+    "MapOneToOne",
+]

cognite/neat/rules/transformers/_base.py

@@ -0,0 +1,81 @@
+from abc import ABC, abstractmethod
+from collections.abc import MutableSequence
+from typing import Generic, TypeVar
+
+from cognite.neat.issues import IssueList, NeatError
+from cognite.neat.issues.errors import NeatTypeError, NeatValueError
+from cognite.neat.rules._shared import (
+    InputRules,
+    JustRules,
+    MaybeRules,
+    OutRules,
+    Rules,
+    VerifiedRules,
+)
+
+T_RulesIn = TypeVar("T_RulesIn", bound=Rules)
+T_RulesOut = TypeVar("T_RulesOut", bound=Rules)
+
+
+class RulesTransformer(ABC, Generic[T_RulesIn, T_RulesOut]):
+    """This is the base class for all rule transformers.
+
+    Note transformers follow the functional pattern Monad
+    https://en.wikipedia.org/wiki/Monad_(functional_programming)
+    """
+
+    @abstractmethod
+    def transform(self, rules: T_RulesIn | OutRules[T_RulesIn]) -> OutRules[T_RulesOut]:
+        """Transform the input rules into the output rules."""
+        raise NotImplementedError()
+
+    def try_transform(self, rules: MaybeRules[T_RulesIn]) -> MaybeRules[T_RulesOut]:
+        """Try to transform the input rules into the output rules."""
+        try:
+            result = self.transform(rules)
+        except NeatError:
+            # Any error caught during transformation will be returned as issues
+            return MaybeRules(None, rules.issues)
+        issues = IssueList(rules.issues, title=rules.issues.title)
+        if isinstance(result, MaybeRules):
+            issues.extend(result.issues)
+        return MaybeRules(result.get_rules(), issues)
+
+    @classmethod
+    def _to_rules(cls, rules: T_RulesIn | OutRules[T_RulesIn]) -> T_RulesIn:
+        if isinstance(rules, JustRules):
+            return rules.rules
+        elif isinstance(rules, MaybeRules):
+            if rules.rules is None:
+                raise NeatValueError("Rules is missing cannot convert")
+            return rules.rules
+        elif isinstance(rules, VerifiedRules | InputRules):
+            return rules  # type: ignore[return-value]
+        else:
+            raise NeatTypeError(f"Unsupported type: {type(rules)}")
+
+
+class RulesPipeline(list, MutableSequence[RulesTransformer], Generic[T_RulesIn, T_RulesOut]):
+    def transform(self, rules: T_RulesIn | OutRules[T_RulesIn]) -> OutRules[T_RulesOut]:
+        """Transform the input rules into the output rules."""
+        for transformer in self:
+            rules = transformer.transform(rules)
+        return rules  # type: ignore[return-value]
+
+    def try_transform(self, rules: MaybeRules[T_RulesIn]) -> MaybeRules[T_RulesOut]:
+        """Try to transform the input rules into the output rules."""
+        for transformer in self:
+            rules = transformer.try_transform(rules)
+        return rules  # type: ignore[return-value]
+
+    def run(self, rules: T_RulesIn | OutRules[T_RulesIn]) -> T_RulesOut:
+        """Run the pipeline from the input rules to the output rules."""
+        output = self.transform(rules)
+        if isinstance(output, MaybeRules):
+            if output.rules is None:
+                raise NeatValueError(f"Rule transformation failed: {output.issues}")
+            return output.rules
+        elif isinstance(output, JustRules):
+            return output.rules
+        else:
+            raise NeatTypeError(f"Rule transformation failed: {output}")