cognite-neat 0.72.2__py3-none-any.whl → 0.73.0__py3-none-any.whl

cognite/neat/rules/importers/_spreadsheet2rules.py

@@ -4,6 +4,7 @@ generating a list of rules based on which nodes that form the graph are made.
 """
 
 from collections import UserDict, defaultdict
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Literal, cast, overload
 
@@ -49,6 +50,12 @@ class MetadataRaw(UserDict):
     def has_schema_field(self) -> bool:
         return self.get("schema") in [schema.value for schema in SchemaCompleteness.__members__.values()]
 
+    @property
+    def schema(self) -> SchemaCompleteness | None:
+        if not self.has_schema_field:
+            return None
+        return SchemaCompleteness(self["schema"])
+
     def is_valid(self, issue_list: IssueList, filepath: Path) -> bool:
         if not self.has_role_field:
             issue_list.append(issues.spreadsheet_file.RoleMissingOrUnsupportedError(filepath))
@@ -61,6 +68,14 @@ class MetadataRaw(UserDict):
         return True
 
 
+@dataclass
+class ReadResult:
+    sheets: dict[str, dict | list]
+    read_info_by_sheet: dict[str, SpreadsheetRead]
+    role: RoleTypes
+    schema: SchemaCompleteness | None
+
+
 class SpreadsheetReader:
     def __init__(self, issue_list: IssueList, is_reference: bool = False):
         self.issue_list = issue_list
@@ -79,7 +94,7 @@ class SpreadsheetReader:
     def to_reference_sheet(cls, sheet_name: str) -> str:
         return f"Ref{sheet_name}"
 
-    def read(self, filepath: Path) -> Rules | None:
+    def read(self, filepath: Path) -> None | ReadResult:
         with pd.ExcelFile(filepath) as excel_file:
             if self.metadata_sheet_name not in excel_file.sheet_names:
                 self.issue_list.append(
@@ -95,21 +110,10 @@ class SpreadsheetReader:
                 return None
 
             sheets, read_info_by_sheet = self._read_sheets(metadata, excel_file)
-            if self.issue_list.has_errors:
-                return None
-
-            rules_cls = RULES_PER_ROLE[metadata.role]
-            with _handle_issues(
-                self.issue_list,
-                error_cls=issues.spreadsheet.InvalidSheetError,
-                error_args={"read_info_by_sheet": read_info_by_sheet},
-            ) as future:
-                rules = rules_cls.model_validate(sheets)  # type: ignore[attr-defined]
-
-            if future.result == "failure" or self.issue_list.has_errors:
+            if sheets is None or self.issue_list.has_errors:
                 return None
 
-        return rules
+            return ReadResult(sheets, read_info_by_sheet, metadata.role, metadata.schema)
 
     def _read_sheets(
         self, metadata: MetadataRaw, excel_file: ExcelFile
@@ -167,43 +171,60 @@ class ExcelImporter(BaseImporter):
         is_reference: bool = False,
     ) -> tuple[Rules | None, IssueList] | Rules:
         issue_list = IssueList(title=f"'{self.filepath.name}'")
-
         if not self.filepath.exists():
             issue_list.append(issues.spreadsheet_file.SpreadsheetNotFoundError(self.filepath))
             return self._return_or_raise(issue_list, errors)
 
-        user_rules: Rules | None = None
+        user_result: ReadResult | None = None
         if not is_reference:
-            user_rules = SpreadsheetReader(issue_list, is_reference=False).read(self.filepath)
-            if issue_list.has_errors:
+            user_result = SpreadsheetReader(issue_list, is_reference=False).read(self.filepath)
+            if user_result is None or issue_list.has_errors:
                 return self._return_or_raise(issue_list, errors)
 
-        reference_rules: Rules | None = None
+        reference_result: ReadResult | None = None
         if is_reference or (
-            user_rules
-            and user_rules.metadata.role != RoleTypes.domain_expert
-            and cast(DMSRules | InformationRules, user_rules).metadata.schema_ == SchemaCompleteness.extended
+            user_result
+            and user_result.role != RoleTypes.domain_expert
+            and user_result.schema == SchemaCompleteness.extended
         ):
-            reference_rules = SpreadsheetReader(issue_list, is_reference=True).read(self.filepath)
+            reference_result = SpreadsheetReader(issue_list, is_reference=True).read(self.filepath)
             if issue_list.has_errors:
                 return self._return_or_raise(issue_list, errors)
 
-        if user_rules and reference_rules and user_rules.metadata.role != reference_rules.metadata.role:
+        if user_result and reference_result and user_result.role != reference_result.role:
             issue_list.append(issues.spreadsheet_file.RoleMismatchError(self.filepath))
             return self._return_or_raise(issue_list, errors)
 
-        if user_rules and reference_rules:
-            rules = user_rules
-            rules.reference = reference_rules
-        elif user_rules:
-            rules = user_rules
-        elif reference_rules:
-            rules = reference_rules
+        if user_result and reference_result:
+            user_result.sheets["reference"] = reference_result.sheets
+            sheets = user_result.sheets
+            original_role = user_result.role
+            read_info_by_sheet = user_result.read_info_by_sheet
+            read_info_by_sheet.update(reference_result.read_info_by_sheet)
+        elif user_result:
+            sheets = user_result.sheets
+            original_role = user_result.role
+            read_info_by_sheet = user_result.read_info_by_sheet
+        elif reference_result:
+            sheets = reference_result.sheets
+            original_role = reference_result.role
+            read_info_by_sheet = reference_result.read_info_by_sheet
         else:
             raise ValueError(
                 "No rules were generated. This should have been caught earlier. " f"Bug in {type(self).__name__}."
             )
 
+        rules_cls = RULES_PER_ROLE[original_role]
+        with _handle_issues(
+            issue_list,
+            error_cls=issues.spreadsheet.InvalidSheetError,
+            error_args={"read_info_by_sheet": read_info_by_sheet},
+        ) as future:
+            rules = rules_cls.model_validate(sheets)  # type: ignore[attr-defined]
+
+        if future.result == "failure" or issue_list.has_errors:
+            return self._return_or_raise(issue_list, errors)
+
         return self._to_output(
             rules,
             issue_list,

cognite/neat/rules/issues/base.py

@@ -68,7 +68,15 @@ class NeatValidationError(ValidationIssue, ABC):
 
         This is intended to be overridden in subclasses to handle specific error types.
         """
-        return [DefaultPydanticError.from_pydantic_error(error) for error in errors]
+        all_errors = []
+        for error in errors:
+            if isinstance(ctx := error.get("ctx"), dict) and isinstance(
+                multi_error := ctx.get("error"), MultiValueError
+            ):
+                all_errors.extend(multi_error.errors)
+            else:
+                all_errors.append(DefaultPydanticError.from_pydantic_error(error))
+        return all_errors
 
 
 @dataclass(frozen=True)

cognite/neat/rules/issues/dms.py

@@ -26,6 +26,8 @@ __all__ = [
     "MultipleReferenceWarning",
     "HasDataFilterOnNoPropertiesViewWarning",
     "NodeTypeFilterOnParentViewWarning",
+    "ChangingContainerError",
+    "ChangingViewError",
 ]
 
 
@@ -225,6 +227,78 @@ class ContainerPropertyUsedMultipleTimesError(DMSSchemaError):
         return output
 
 
+@dataclass(frozen=True)
+class ChangingContainerError(DMSSchemaError):
+    description = "You are adding to an existing model. "
+    fix = "Keep the container the same"
+    error_name: ClassVar[str] = "ChangingContainerError"
+    container_id: dm.ContainerId
+    changed_properties: list[str] | None = None
+    changed_attributes: list[str] | None = None
+
+    def __post_init__(self):
+        # Sorting for deterministic output
+        if self.changed_properties:
+            self.changed_properties.sort()
+        if self.changed_attributes:
+            self.changed_attributes.sort()
+
+    def message(self) -> str:
+        if self.changed_properties:
+            changed = f" properties {self.changed_properties}."
+        elif self.changed_attributes:
+            changed = f" attributes {self.changed_attributes}."
+        else:
+            changed = "."
+        return (
+            f"The container {self.container_id} has changed{changed}"
+            "When extending model with extension set to addition or reshape, the container "
+            "properties must remain the same"
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["container_id"] = self.container_id.dump()
+        output["changed_properties"] = self.changed_properties
+        return output
+
+
+@dataclass(frozen=True)
+class ChangingViewError(DMSSchemaError):
+    description = "You are adding to an existing model. "
+    fix = "Keep the view the same"
+    error_name: ClassVar[str] = "ChangingViewError"
+    view_id: dm.ViewId
+    changed_properties: list[str] | None = None
+    changed_attributes: list[str] | None = None
+
+    def __post_init__(self):
+        # Sorting for deterministic output
+        if self.changed_properties:
+            self.changed_properties.sort()
+        if self.changed_attributes:
+            self.changed_attributes.sort()
+
+    def message(self) -> str:
+        if self.changed_properties:
+            changed = f" properties {self.changed_properties}."
+        elif self.changed_attributes:
+            changed = f" attributes {self.changed_attributes}."
+        else:
+            changed = "."
+
+        return (
+            f"The view {self.view_id} has changed{changed}"
+            "When extending model with extension set to addition, the view properties must remain the same"
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["view_id"] = self.view_id.dump()
+        output["difference"] = self.changed_properties
+        return output
+
+
 @dataclass(frozen=True)
 class DirectRelationListWarning(DMSSchemaWarning):
     description = "The container property is set to a direct relation list, which is not supported by the CDF API"

cognite/neat/rules/models/_rules/_types/_base.py

@@ -14,7 +14,6 @@ from cognite.neat.rules.models.rdfpath import (
     TransformationRuleType,
     parse_rule,
 )
-from cognite.neat.utils.text import to_pascal
 
 if sys.version_info >= (3, 11):
     from enum import StrEnum
@@ -197,15 +196,14 @@ class ContainerEntity(Entity):
     def from_id(cls, container_id: ContainerId) -> "ContainerEntity":
         return ContainerEntity(prefix=container_id.space, suffix=container_id.external_id)
 
-    def as_id(self, default_space: str | None, standardize_casing: bool = True) -> ContainerId:
+    def as_id(self, default_space: str | None) -> ContainerId:
         if self.space is Undefined and default_space is None:
             raise ValueError("Space is Undefined! Set default_space!")
 
-        external_id = to_pascal(self.external_id) if standardize_casing else self.external_id
         if self.space is Undefined:
-            return ContainerId(space=cast(str, default_space), external_id=external_id)
+            return ContainerId(space=cast(str, default_space), external_id=self.external_id)
         else:
-            return ContainerId(space=self.space, external_id=external_id)
+            return ContainerId(space=self.space, external_id=self.external_id)
 
 
 class ViewEntity(Entity):
@@ -233,7 +231,6 @@ class ViewEntity(Entity):
         allow_none: Literal[False] = False,
         default_space: str | None = None,
         default_version: str | None = None,
-        standardize_casing: bool = True,
     ) -> ViewId:
         ...
 
@@ -243,7 +240,6 @@ class ViewEntity(Entity):
         allow_none: Literal[True],
         default_space: str | None = None,
         default_version: str | None = None,
-        standardize_casing: bool = True,
     ) -> ViewId | None:
         ...
 
@@ -252,7 +248,6 @@ class ViewEntity(Entity):
         allow_none: bool = False,
         default_space: str | None = None,
         default_version: str | None = None,
-        standardize_casing: bool = True,
     ) -> ViewId | None:
         if self.suffix is Unknown and allow_none:
             return None
@@ -265,8 +260,7 @@ class ViewEntity(Entity):
             raise ValueError("space is required")
         if version is None:
             raise ValueError("version is required")
-        external_id = to_pascal(self.external_id) if standardize_casing else self.external_id
-        return ViewId(space=space, external_id=external_id, version=version)
+        return ViewId(space=space, external_id=self.external_id, version=version)
 
 
 class ViewPropEntity(ViewEntity):
@@ -303,14 +297,10 @@ class ViewPropEntity(ViewEntity):
             property_=prop_id.property,
         )
 
-    def as_prop_id(
-        self, default_space: str | None = None, default_version: str | None = None, standardize_casing: bool = True
-    ) -> PropertyId:
+    def as_prop_id(self, default_space: str | None = None, default_version: str | None = None) -> PropertyId:
         if self.property_ is None:
             raise ValueError("property is required to create PropertyId")
-        return PropertyId(
-            source=self.as_id(False, default_space, default_version, standardize_casing), property=self.property_
-        )
+        return PropertyId(source=self.as_id(False, default_space, default_version), property=self.property_)
 
     @property
     def versioned_id(self) -> str:

cognite/neat/rules/models/_rules/base.py

@@ -10,10 +10,21 @@ import types
 from abc import abstractmethod
 from collections.abc import Callable, Iterator
 from functools import wraps
-from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
+from typing import Annotated, Any, ClassVar, Generic, TypeAlias, TypeVar
 
 import pandas as pd
-from pydantic import BaseModel, ConfigDict, Field, HttpUrl, constr, field_validator, model_serializer, model_validator
+from pydantic import (
+    BaseModel,
+    BeforeValidator,
+    ConfigDict,
+    Field,
+    HttpUrl,
+    PlainSerializer,
+    constr,
+    field_validator,
+    model_serializer,
+    model_validator,
+)
 from pydantic.fields import FieldInfo
 
 from cognite.neat.rules.models._rules._types import ClassType
@@ -318,3 +329,14 @@ class SheetList(BaseModel, Generic[T_Entity]):
     def mandatory_fields(cls, use_alias=False) -> set[str]:
         """Returns a set of mandatory fields for the model."""
         return _get_required_fields(cls, use_alias)
+
+
+ExtensionCategoryType = Annotated[
+    ExtensionCategory,
+    PlainSerializer(
+        lambda v: v.value if isinstance(v, ExtensionCategory) else v,
+        return_type=str,
+        when_used="unless-none",
+    ),
+    BeforeValidator(lambda v: ExtensionCategory(v) if isinstance(v, str) else v),
+]