cognite-neat 0.88.2__py3-none-any.whl → 0.89.0__py3-none-any.whl

cognite/neat/rules/models/_base_input.py

@@ -0,0 +1,162 @@
+"""Module for base classes for the input models.
+
+The philosophy of the input models is:
+
+* Provide an easy way to input rules. The type hints are made to be human-friendly, for example, Literal instead of
+  Enum.
+* The .dump() method should fill out defaults and have shortcuts. For example, if the prefix is not provided for
+  a class, then the prefix from the metadata is used. For views, if the class is not provided, it is assumed to
+  be the same as the view.
+
+The base classes are to make it easy to create the input models with default behavior. They are also used for
+testing to ensure that input models correctly map to the verified rules models.
+"""
+
+import sys
+from abc import ABC, abstractmethod
+from dataclasses import Field, dataclass, fields, is_dataclass
+from types import GenericAlias, UnionType
+from typing import Any, Generic, TypeVar, Union, cast, get_args, get_origin, overload
+
+from ._base_rules import BaseRules, RuleModel
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+T_BaseRules = TypeVar("T_BaseRules", bound=BaseRules)
+T_RuleModel = TypeVar("T_RuleModel", bound=RuleModel)
+
+
+@dataclass
+class InputRules(Generic[T_BaseRules], ABC):
+    """Input rules are raw data that is not yet validated."""
+
+    @classmethod
+    @abstractmethod
+    def _get_verified_cls(cls) -> type[T_BaseRules]:
+        raise NotImplementedError("This method should be implemented in the subclass.")
+
+    @classmethod
+    @overload
+    def load(cls, data: dict[str, Any]) -> Self: ...
+
+    @classmethod
+    @overload
+    def load(cls, data: None) -> None: ...
+
+    @classmethod
+    def load(cls, data: dict | None) -> Self | None:
+        if data is None:
+            return None
+        return cls._load(data)
+
+    @classmethod
+    def _type_by_field_name(cls) -> dict[str, type]:
+        output: dict[str, type] = {}
+        for field_ in fields(cls):
+            type_ = field_.type
+            if isinstance(type_, UnionType) or get_origin(type_) is Union:
+                type_ = get_args(type_)[0]
+            if isinstance(type_, str) and type_.startswith(cls.__name__):
+                type_ = cls
+
+            if is_dataclass(type_):
+                candidate = type_
+            elif isinstance(type_, GenericAlias) and type_.__origin__ is list and is_dataclass(type_.__args__[0]):
+                candidate = type_.__args__[0]
+            else:
+                continue
+
+            if hasattr(candidate, "_load"):
+                output[field_.name] = candidate
+        return output
+
+    @classmethod
+    def _load(cls, data: dict[str, Any]) -> Self:
+        args: dict[str, Any] = {}
+        field_type_by_name = cls._type_by_field_name()
+        for field_name, field_ in cls._get_verified_cls().model_fields.items():
+            field_type = field_type_by_name.get(field_name)
+            if field_type is None:
+                continue
+            if field_name in data:
+                value = data[field_name]
+            elif field_.alias in data:
+                value = data[field_.alias]
+            else:
+                continue
+
+            if isinstance(value, dict):
+                args[field_name] = field_type._load(value)  # type: ignore[attr-defined]
+            elif isinstance(value, list) and value and isinstance(value[0], dict):
+                args[field_name] = [field_type._load(item) for item in value]  # type: ignore[attr-defined]
+        return cls(**args)
+
+    def _dataclass_fields(self) -> list[Field]:
+        return list(fields(self))
+
+    def as_rules(self) -> T_BaseRules:
+        cls_ = self._get_verified_cls()
+        return cls_.model_validate(self.dump())
+
+    def dump(self) -> dict[str, Any]:
+        output: dict[str, Any] = {}
+        for field_ in self._dataclass_fields():
+            value = getattr(self, field_.name)
+            if value is None:
+                continue
+            if hasattr(value, "dump"):
+                output[field_.name] = value.dump()
+            elif isinstance(value, list) and value and hasattr(value[0], "dump"):
+                output[field_.name] = [item.dump() for item in value]
+        return output
+
+
+@dataclass
+class InputComponent(ABC, Generic[T_RuleModel]):
+    @classmethod
+    @abstractmethod
+    def _get_verified_cls(cls) -> type[T_RuleModel]:
+        raise NotImplementedError("This method should be implemented in the subclass.")
+
+    @classmethod
+    @overload
+    def load(cls, data: None) -> None: ...
+
+    @classmethod
+    @overload
+    def load(cls, data: dict[str, Any]) -> Self: ...
+
+    @classmethod
+    @overload
+    def load(cls, data: list[dict[str, Any]]) -> list[Self]: ...
+
+    @classmethod
+    def load(cls, data: dict[str, Any] | list[dict[str, Any]] | None) -> Self | list[Self] | None:
+        if data is None:
+            return None
+        if isinstance(data, list) or (isinstance(data, dict) and isinstance(data.get("data"), list)):
+            items = cast(list[dict[str, Any]], data.get("data") if isinstance(data, dict) else data)
+            return [loaded for item in items if (loaded := cls.load(item)) is not None]
+        return cls._load(data)
+
+    @classmethod
+    def _load(cls, data: dict[str, Any]) -> Self:
+        args: dict[str, Any] = {}
+        for field_name, field_ in cls._get_verified_cls().model_fields.items():  # type: ignore[attr-defined]
+            if field_.exclude:
+                continue
+            if field_name in data:
+                args[field_name] = data[field_name]
+            elif field_.alias in data:
+                args[field_name] = data[field_.alias]
+        return cls(**args)
+
+    def dump(self, **kwargs) -> dict[str, Any]:
+        return {
+            field_.alias or name: getattr(self, name)
+            for name, field_ in self._get_verified_cls().model_fields.items()
+            if not field_.exclude
+        }

cognite/neat/rules/models/{_base.py → _base_rules.py}

@@ -10,7 +10,7 @@ import types
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Iterator
 from functools import wraps
-from typing import Annotated, Any, ClassVar, Generic, Literal, TypeAlias, TypeVar
+from typing import Annotated, Any, ClassVar, Generic, Literal, TypeVar
 
 import pandas as pd
 from pydantic import (
@@ -19,7 +19,6 @@ from pydantic import (
     ConfigDict,
     Field,
     PlainSerializer,
-    constr,
     field_validator,
     model_serializer,
     model_validator,
@@ -36,12 +35,6 @@ else:
 METADATA_VALUE_MAX_LENGTH = 5120
 
 
-def _add_alias(data: dict[str, Any], base_model: type[BaseModel]) -> None:
-    for field_name, field_ in base_model.model_fields.items():
-        if field_name not in data and field_.alias in data:
-            data[field_name] = data[field_.alias]
-
-
 def replace_nan_floats_with_default(values: dict, model_fields: dict[str, FieldInfo]) -> dict:
     output = {}
     for field_name, value in values.items():
@@ -126,10 +119,6 @@ def _get_required_fields(model: type[BaseModel], use_alias: bool = False) -> set
     return required_fields
 
 
-Space: TypeAlias = str
-Description: TypeAlias = constr(min_length=1, max_length=1024)  # type: ignore[valid-type]
-
-
 class SchemaCompleteness(StrEnum):
     complete = "complete"
     partial = "partial"

cognite/neat/rules/models/_rdfpath.py

@@ -8,7 +8,7 @@ from typing import ClassVar, Literal
 
 from pydantic import BaseModel, field_validator, model_serializer
 
-from cognite.neat.rules.issues.tables import NotValidRAWLookUpError, NotValidRDFPathError, NotValidTableLookUpError
+from cognite.neat.issues.errors import NeatValueError
 
 if sys.version_info >= (3, 11):
     from enum import StrEnum
@@ -313,7 +313,7 @@ def parse_traversal(raw: str) -> SelfReferenceProperty | SingleProperty | Hop:
     elif result := HOP_REGEX_COMPILED.match(raw):
         return Hop.from_string(class_=result.group("origin"), traversal=result.group(_traversal))
     else:
-        raise NotValidRDFPathError(raw).as_pydantic_exception()
+        raise NeatValueError(f"Invalid RDF Path: {raw!r}")
 
 
 def parse_table_lookup(raw: str) -> TableLookup:
@@ -323,7 +323,7 @@ def parse_table_lookup(raw: str) -> TableLookup:
             key=result.group(Lookup.key),
             value=result.group(Lookup.value),
         )
-    raise NotValidTableLookUpError(raw).as_pydantic_exception()
+    raise NeatValueError(f"Invalid table lookup: {raw!r}")
 
 
 def parse_rule(rule_raw: str, rule_type: TransformationRuleType | None) -> RDFPath:
@@ -334,7 +334,7 @@ def parse_rule(rule_raw: str, rule_type: TransformationRuleType | None) -> RDFPa
         case TransformationRuleType.rawlookup:
             rule_raw = rule_raw.replace(" ", "")
             if Counter(rule_raw).get("|") != 1:
-                raise NotValidRAWLookUpError(rule_raw).as_pydantic_exception()
+                raise NeatValueError(f"Invalid rawlookup rule: {rule_raw!r}")
             traversal, table_lookup = rule_raw.split("|")
             return RawLookup(
                 traversal=parse_traversal(traversal),

cognite/neat/rules/models/{_types/_field.py → _types.py}

@@ -14,10 +14,9 @@ from pydantic import (
     WrapValidator,
 )
 from pydantic.functional_serializers import PlainSerializer
-from pydantic_core import PydanticCustomError
 
-from cognite.neat.issues.neat_warnings.identifier import RegexViolationWarning
-from cognite.neat.rules.issues.spreadsheet import RegexViolationError
+from cognite.neat.issues.errors import RegexViolationError
+from cognite.neat.issues.warnings import RegexViolationWarning
 from cognite.neat.utils.regex_patterns import (
     PATTERNS,
     PREFIX_COMPLIANCE_REGEX,
@@ -36,10 +35,6 @@ def _custom_error(exc_factory: Callable[[str | None, Exception], Any]) -> Any:
     return WrapValidator(_validator)
 
 
-def _raise(exception: PydanticCustomError):
-    raise exception
-
-
 StrOrListType = Annotated[
     str | list[str],
     BeforeValidator(lambda value: value.replace(", ", ",").split(",") if isinstance(value, str) and value else value),
@@ -72,7 +67,7 @@ NamespaceType = Annotated[
 PrefixType = Annotated[
     str,
     StringConstraints(pattern=PREFIX_COMPLIANCE_REGEX),
-    _custom_error(lambda _, value: RegexViolationError(value, PREFIX_COMPLIANCE_REGEX).as_pydantic_exception()),
+    _custom_error(lambda _, value: RegexViolationError(value, PREFIX_COMPLIANCE_REGEX)),
 ]
 
 ExternalIdType = Annotated[
@@ -83,13 +78,13 @@ ExternalIdType = Annotated[
 VersionType = Annotated[
     str,
     StringConstraints(pattern=VERSION_COMPLIANCE_REGEX),
-    _custom_error(lambda _, value: RegexViolationError(value, VERSION_COMPLIANCE_REGEX).as_pydantic_exception()),
+    _custom_error(lambda _, value: RegexViolationError(value, VERSION_COMPLIANCE_REGEX)),
 ]
 
 
 def _property_validation(value: str) -> str:
     if not PATTERNS.property_id_compliance.match(value):
-        _raise(RegexViolationError(value, PROPERTY_ID_COMPLIANCE_REGEX).as_pydantic_exception())
+        raise RegexViolationError(value, PROPERTY_ID_COMPLIANCE_REGEX)
     if PATTERNS.more_than_one_alphanumeric.search(value):
         warnings.warn(
             RegexViolationWarning(value, PROPERTY_ID_COMPLIANCE_REGEX, "property", "MoreThanOneNonAlphanumeric"),

cognite/neat/rules/models/asset/__init__.py

@@ -1,10 +1,13 @@
 from ._rules import AssetClass, AssetMetadata, AssetProperty, AssetRules
-from ._rules_input import AssetRulesInput
+from ._rules_input import AssetInputClass, AssetInputMetadata, AssetInputProperty, AssetInputRules
 
 __all__ = [
     "AssetRules",
     "AssetMetadata",
     "AssetClass",
     "AssetProperty",
-    "AssetRulesInput",
+    "AssetInputRules",
+    "AssetInputMetadata",
+    "AssetInputClass",
+    "AssetInputProperty",
 ]

cognite/neat/rules/models/asset/_rules.py

@@ -1,15 +1,12 @@
 import sys
-from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
+from typing import Any, ClassVar, Literal, cast
 
 from pydantic import Field, field_validator, model_validator
 from pydantic.main import IncEx
 from rdflib import Namespace
 
 from cognite.neat.constants import get_default_prefixes
-from cognite.neat.issues import MultiValueError
-from cognite.neat.rules import issues
-from cognite.neat.rules.models._base import BaseRules, RoleTypes, SheetList
-from cognite.neat.rules.models.domain import DomainRules
+from cognite.neat.rules.models._base_rules import BaseRules, RoleTypes, SheetList
 from cognite.neat.rules.models.entities import (
     CdfResourceEntityList,
     ClassEntity,
@@ -23,10 +20,6 @@ from cognite.neat.rules.models.information import (
     InformationRules,
 )
 
-if TYPE_CHECKING:
-    from cognite.neat.rules.models.dms._rules import DMSRules
-
-
 if sys.version_info >= (3, 11):
     from typing import Self
 else:
@@ -109,7 +102,7 @@ class AssetRules(BaseRules):
         if issue_list.warnings:
             issue_list.trigger_warnings()
         if issue_list.has_errors:
-            raise MultiValueError([error for error in issue_list if isinstance(error, issues.NeatValidationError)])
+            raise issue_list.as_exception()
         return self
 
     def dump(
@@ -143,16 +136,3 @@ class AssetRules(BaseRules):
             prefix = self.reference.metadata.prefix
             cleaned[reference] = _AssetRulesSerializer(by_alias, prefix).clean(ref_dump, True)
         return cleaned
-
-    def as_domain_rules(self) -> DomainRules:
-        from ._converter import _AssetRulesConverter
-
-        return _AssetRulesConverter(self.as_information_rules()).as_domain_rules()
-
-    def as_dms_rules(self) -> "DMSRules":
-        from ._converter import _AssetRulesConverter
-
-        return _AssetRulesConverter(self.as_information_rules()).as_dms_rules()
-
-    def as_information_rules(self) -> InformationRules:
-        return InformationRules.model_validate(self.model_dump())

cognite/neat/rules/models/asset/_rules_input.py

@@ -1,29 +1,33 @@
-from collections.abc import Sequence
 from dataclasses import dataclass
-from typing import Any, cast, overload
+from typing import Any
 
-from cognite.neat.rules.models._base import _add_alias
+from rdflib import Namespace
+
+from cognite.neat.rules.models._base_input import InputComponent, InputRules
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import (
     ClassEntity,
     MultiValueTypeInfo,
-    Unknown,
     UnknownEntity,
+    load_value_type,
 )
-from cognite.neat.rules.models.information._rules_input import InformationClassInput, InformationMetadataInput
+from cognite.neat.rules.models.information._rules_input import InformationInputClass, InformationInputMetadata
 
-from ._rules import AssetProperty, AssetRules
+from ._rules import AssetClass, AssetMetadata, AssetProperty, AssetRules
 
 
 @dataclass
-class AssetMetadataInput(InformationMetadataInput): ...
+class AssetInputMetadata(InformationInputMetadata):
+    @classmethod
+    def _get_verified_cls(cls) -> type[AssetMetadata]:
+        return AssetMetadata
 
 
 @dataclass
-class AssetPropertyInput:
-    class_: str
+class AssetInputProperty(InputComponent[AssetProperty]):
+    class_: ClassEntity | str
     property_: str
-    value_type: str
+    value_type: DataType | ClassEntity | MultiValueTypeInfo | UnknownEntity | str
     name: str | None = None
     description: str | None = None
     comment: str | None = None
@@ -34,133 +38,54 @@ class AssetPropertyInput:
     match_type: str | None = None
     transformation: str | None = None
     implementation: str | None = None
+    # Only used internally
+    inherited: bool = False
 
     @classmethod
-    @overload
-    def load(cls, data: None) -> None: ...
+    def _get_verified_cls(cls) -> type[AssetProperty]:
+        return AssetProperty
 
-    @classmethod
-    @overload
-    def load(cls, data: dict[str, Any]) -> "AssetPropertyInput": ...
+    def dump(self, default_prefix: str) -> dict[str, Any]:  # type: ignore[override]
+        output = super().dump()
+        output["Class"] = ClassEntity.load(self.class_, prefix=default_prefix)
+        output["Value Type"] = load_value_type(self.value_type, default_prefix)
+        return output
 
-    @classmethod
-    @overload
-    def load(cls, data: list[dict[str, Any]]) -> list["AssetPropertyInput"]: ...
 
+@dataclass
+class AssetInputClass(InformationInputClass):
     @classmethod
-    def load(
-        cls, data: dict[str, Any] | list[dict[str, Any]] | None
-    ) -> "AssetPropertyInput | list[AssetPropertyInput] | None":
-        if data is None:
-            return None
-        if isinstance(data, list) or (isinstance(data, dict) and isinstance(data.get("data"), list)):
-            items = cast(list[dict[str, Any]], data.get("data") if isinstance(data, dict) else data)
-            return [loaded for item in items if (loaded := cls.load(item)) is not None]
-
-        _add_alias(data, AssetProperty)
-        return cls(
-            class_=data.get("class_"),  # type: ignore[arg-type]
-            property_=data.get("property_"),  # type: ignore[arg-type]
-            name=data.get("name", None),
-            description=data.get("description", None),
-            comment=data.get("comment", None),
-            value_type=data.get("value_type"),  # type: ignore[arg-type]
-            min_count=data.get("min_count", None),
-            max_count=data.get("max_count", None),
-            default=data.get("default", None),
-            reference=data.get("reference", None),
-            match_type=data.get("match_type", None),
-            transformation=data.get("transformation", None),
-            implementation=data.get("implementation", None),
-        )
-
-    def dump(self, default_prefix: str) -> dict[str, Any]:
-        value_type: MultiValueTypeInfo | DataType | ClassEntity | UnknownEntity
-
-        # property holding xsd data type
-        # check if it is multi value type
-        if "|" in self.value_type:
-            value_type = MultiValueTypeInfo.load(self.value_type)
-            value_type.set_default_prefix(default_prefix)
-
-        elif DataType.is_data_type(self.value_type):
-            value_type = DataType.load(self.value_type)
-
-        # unknown value type
-        elif self.value_type == str(Unknown):
-            value_type = UnknownEntity()
-
-        # property holding link to class
-        else:
-            value_type = ClassEntity.load(self.value_type, prefix=default_prefix)
-
-        return {
-            "Class": ClassEntity.load(self.class_, prefix=default_prefix),
-            "Property": self.property_,
-            "Name": self.name,
-            "Description": self.description,
-            "Comment": self.comment,
-            "Value Type": value_type,
-            "Min Count": self.min_count,
-            "Max Count": self.max_count,
-            "Default": self.default,
-            "Reference": self.reference,
-            "Match Type": self.match_type,
-            "Transformation": self.transformation,
-            "Implementation": self.implementation,
-        }
-
-
-class AssetClassInput(InformationClassInput): ...
+    def _get_verified_cls(cls) -> type[AssetClass]:
+        return AssetClass
 
 
 @dataclass
-class AssetRulesInput:
-    metadata: AssetMetadataInput
-    properties: Sequence[AssetPropertyInput]
-    classes: Sequence[AssetClassInput]
-    last: "AssetRulesInput | AssetRules | None" = None
-    reference: "AssetRulesInput | AssetRules | None" = None
-
-    @classmethod
-    @overload
-    def load(cls, data: dict[str, Any]) -> "AssetRulesInput": ...
-
-    @classmethod
-    @overload
-    def load(cls, data: None) -> None: ...
+class AssetInputRules(InputRules[AssetRules]):
+    metadata: AssetInputMetadata
+    properties: list[AssetInputProperty]
+    classes: list[AssetInputClass]
+    prefixes: dict[str, Namespace] | None = None
+    last: "AssetInputRules | None" = None
+    reference: "AssetInputRules | None" = None
 
     @classmethod
-    def load(cls, data: dict | None) -> "AssetRulesInput | None":
-        if data is None:
-            return None
-        _add_alias(data, AssetRules)
-
-        return cls(
-            metadata=AssetMetadataInput.load(data.get("metadata")),  # type: ignore[arg-type]
-            properties=AssetPropertyInput.load(data.get("properties")),  # type: ignore[arg-type]
-            classes=InformationClassInput.load(data.get("classes")),  # type: ignore[arg-type]
-            last=AssetRulesInput.load(data.get("last")),
-            reference=AssetRulesInput.load(data.get("reference")),
-        )
-
-    def as_rules(self) -> AssetRules:
-        return AssetRules.model_validate(self.dump())
+    def _get_verified_cls(cls) -> type[AssetRules]:
+        return AssetRules
 
     def dump(self) -> dict[str, Any]:
         default_prefix = self.metadata.prefix
         reference: dict[str, Any] | None = None
-        if isinstance(self.reference, AssetRulesInput):
+        if isinstance(self.reference, AssetInputRules):
             reference = self.reference.dump()
         elif isinstance(self.reference, AssetRules):
             # We need to load through the AssetRulesInput to set the correct default space and version
-            reference = AssetRulesInput.load(self.reference.model_dump()).dump()
+            reference = AssetInputRules.load(self.reference.model_dump()).dump()
         last: dict[str, Any] | None = None
-        if isinstance(self.last, AssetRulesInput):
+        if isinstance(self.last, AssetInputRules):
             last = self.last.dump()
         elif isinstance(self.last, AssetRules):
             # We need to load through the AssetRulesInput to set the correct default space and version
-            last = AssetRulesInput.load(self.last.model_dump()).dump()
+            last = AssetInputRules.load(self.last.model_dump()).dump()
 
         return dict(
             Metadata=self.metadata.dump(),

cognite/neat/rules/models/asset/_validation.py

@@ -2,8 +2,8 @@ from graphlib import CycleError
 from typing import cast
 
 from cognite.neat.issues import IssueList
-from cognite.neat.rules import issues
-from cognite.neat.rules.models._base import SheetList
+from cognite.neat.issues.errors import NeatValueError, PropertyDefinitionError
+from cognite.neat.rules.models._base_rules import SheetList
 from cognite.neat.rules.models.asset._rules import AssetProperty, AssetRules
 from cognite.neat.rules.models.entities import AssetEntity, AssetFields, ClassEntity
 from cognite.neat.rules.models.information._validation import InformationPostValidation
@@ -17,7 +17,6 @@ class AssetPostValidation(InformationPostValidation):
         return self.issue_list
 
     def _parent_property_point_to_class(self) -> None:
-        class_property_with_data_value_type = []
         for property_ in cast(SheetList[AssetProperty], self.properties):
             for implementation in property_.implementation:
                 if (
@@ -25,12 +24,15 @@ class AssetPostValidation(InformationPostValidation):
                     and implementation.property_ == AssetFields.parentExternalId
                     and not isinstance(property_.value_type, ClassEntity)
                 ):
-                    class_property_with_data_value_type.append((property_.class_.suffix, property_.property_))
-
-        if class_property_with_data_value_type:
-            self.issue_list.append(
-                issues.spreadsheet.AssetParentPropertyPointsToDataValueTypeError(class_property_with_data_value_type)
-            )
+                    self.issue_list.append(
+                        PropertyDefinitionError(
+                            property_.class_,
+                            "class",
+                            property_.property_,
+                            "parentExternalId is only allowed to "
+                            f"point to a Class not {type(property_.value_type).__name__}",
+                        )
+                    )
 
     def _circular_dependency(self) -> None:
         from cognite.neat.rules.analysis import AssetAnalysis
@@ -38,4 +40,6 @@ class AssetPostValidation(InformationPostValidation):
         try:
             _ = AssetAnalysis(cast(AssetRules, self.rules)).class_topological_sort()
         except CycleError as error:
-            self.issue_list.append(issues.spreadsheet.AssetRulesHaveCircularDependencyError(error.args[1]))
+            self.issue_list.append(
+                NeatValueError(f"Invalid Asset Hierarchy, circular dependency detected: {error.args[1]}")
+            )