cognite-neat 0.75.9__py3-none-any.whl → 0.76.0__py3-none-any.whl

cognite/neat/rules/issues/dms.py

@@ -16,15 +16,16 @@ __all__ = [
     "MissingParentViewError",
     "MissingSourceViewError",
     "MissingEdgeViewError",
-    "DuplicatedViewInDataModelError",
     "DirectRelationMissingSourceWarning",
+    "ViewModelVersionNotMatchingWarning",
+    "ViewModelSpaceNotMatchingWarning",
+    "DuplicatedViewInDataModelError",
     "ContainerPropertyUsedMultipleTimesError",
-    "DirectRelationListWarning",
-    "ReverseOfDirectRelationListWarning",
     "EmptyContainerWarning",
-    "UnsupportedRelationWarning",
+    "UnsupportedConnectionWarning",
     "MultipleReferenceWarning",
     "HasDataFilterOnNoPropertiesViewWarning",
+    "ReverseRelationMissingOtherSideWarning",
     "NodeTypeFilterOnParentViewWarning",
     "ChangingContainerError",
     "ChangingViewError",
@@ -202,6 +203,53 @@ class DirectRelationMissingSourceWarning(DMSSchemaWarning):
         return output
 
 
+@dataclass(frozen=True)
+class ViewModelVersionNotMatchingWarning(DMSSchemaWarning):
+    description = "The view model version does not match the data model version"
+    fix = "Update the view model version to match the data model version"
+    error_name: ClassVar[str] = "ViewModelVersionNotMatching"
+    view_ids: list[dm.ViewId]
+    data_model_version: str
+
+    def message(self) -> str:
+        return (
+            f"The version in the views {self.view_ids} does not match the version in the data model "
+            f"{self.data_model_version}. This is not recommended as it easily leads to confusion and errors. "
+            f"Views are very cheap and we recommend you update the version of the views to match the data"
+            f" model version, irrespective of whether the views have changed or not."
+            " The approach of having same version of model components as the model itself "
+            " is a globally recognized data modeling practice."
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["view_id"] = [view_id.dump() for view_id in self.view_ids]
+        output["data_model_version"] = self.data_model_version
+        return output
+
+
+@dataclass(frozen=True)
+class ViewModelSpaceNotMatchingWarning(DMSSchemaWarning):
+    description = "The view model space does not match the data model space"
+    fix = "Update the view model space to match the data model space"
+    error_name: ClassVar[str] = "ViewModelSpaceNotMatching"
+    view_ids: list[dm.ViewId]
+    data_model_space: str
+
+    def message(self) -> str:
+        return (
+            f"The space in the views {self.view_ids} does not match the space in the data model "
+            f"{self.data_model_space}. This is not recommended as it easily leads to confusion and errors. "
+            f"Views are very cheap and we recommend you always have views in the same space as the data model."
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["view_id"] = [view_id.dump() for view_id in self.view_ids]
+        output["data_model_space"] = self.data_model_space
+        return output
+
+
 @dataclass(frozen=True)
 class ContainerPropertyUsedMultipleTimesError(DMSSchemaError):
     description = "The container property is used multiple times by the same view property"
@@ -298,92 +346,62 @@ class ChangingViewError(DMSSchemaError):
 
 
 @dataclass(frozen=True)
-class DirectRelationListWarning(DMSSchemaWarning):
-    description = "The container property is set to a direct relation list, which is not supported by the CDF API"
-    fix = "Make the property into a multiedge connection instead"
-    error_name: ClassVar[str] = "DirectRelationListWarning"
-    view_id: dm.ViewId
+class EmptyContainerWarning(DMSSchemaWarning):
+    description = "The container is empty"
+    fix = "Add data to the container"
+    error_name: ClassVar[str] = "EmptyContainerWarning"
     container_id: dm.ContainerId
-    property: str
 
     def message(self) -> str:
         return (
-            f"The property in {self.container_id}.{self.property} is a list of direct relations. "
-            f"This is not supported by the API, so it will be converted to an MultiEdgeConnection on"
-            f"the view {self.view_id}.{self.property} instead"
+            f"The container {self.container_id} is empty. Is this intended? Skipping this container "
+            "in the data model."
         )
 
     def dump(self) -> dict[str, Any]:
         output = super().dump()
-        output["view_id"] = self.view_id.dump()
         output["container_id"] = self.container_id.dump()
-        output["property"] = self.property
         return output
 
 
 @dataclass(frozen=True)
-class ReverseOfDirectRelationListWarning(DMSSchemaWarning):
-    description = (
-        "The view property is set to a reverse of a direct relation list, which is not supported by the CDF API"
-    )
-    fix = "Make the property into a multiedge connection instead"
-    error_name: ClassVar[str] = "ReverseOfDirectRelationListWarning"
+class UnsupportedConnectionWarning(DMSSchemaWarning):
+    description = "The connection type is not supported by neat"
+    fix = "Change the connection to a supported type"
+    error_name: ClassVar[str] = "UnsupportedConnectionWarning"
     view_id: dm.ViewId
     property: str
+    connection: str
 
     def message(self) -> str:
         return (
-            f"The property pointed to be {self.view_id}.{self.property} is a list of direct relations. "
-            f"This is not supported by the API, so the {self.view_id}.{self.property} "
-            "will be converted from a reverse direct relation to an MultiEdgeConnection instead"
+            f"The connection {self.connection} in {self.view_id}.{self.property} is not supported."
+            "This property will be ignored."
         )
 
     def dump(self) -> dict[str, Any]:
         output = super().dump()
         output["view_id"] = self.view_id.dump()
         output["property"] = self.property
+        output["connection"] = self.connection
         return output
 
 
 @dataclass(frozen=True)
-class EmptyContainerWarning(DMSSchemaWarning):
-    description = "The container is empty"
-    fix = "Add data to the container"
-    error_name: ClassVar[str] = "EmptyContainerWarning"
-    container_id: dm.ContainerId
-
-    def message(self) -> str:
-        return (
-            f"The container {self.container_id} is empty. Is this intended? Skipping this container "
-            "in the data model."
-        )
-
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["container_id"] = self.container_id.dump()
-        return output
-
-
-@dataclass(frozen=True)
-class UnsupportedRelationWarning(DMSSchemaWarning):
-    description = "The relatio type is not supported by neat"
-    fix = "Change the relation to a supported type"
-    error_name: ClassVar[str] = "UnsupportedRelationWarning"
+class ReverseRelationMissingOtherSideWarning(DMSSchemaWarning):
+    description = "The relation is missing the other side"
+    fix = "Add the other side of the relation"
+    error_name: ClassVar[str] = "ReverseRelationMissingOtherSideWarning"
     view_id: dm.ViewId
     property: str
-    relation: str
 
     def message(self) -> str:
-        return (
-            f"The relation {self.relation} in {self.view_id}.{self.property} is not supported."
-            "This property will be ignored."
-        )
+        return f"The reverse relation specified in {self.view_id}.{self.property} is missing the other side."
 
     def dump(self) -> dict[str, Any]:
         output = super().dump()
         output["view_id"] = self.view_id.dump()
         output["property"] = self.property
-        output["relation"] = self.relation
         return output
 
 
@@ -427,7 +445,7 @@ class HasDataFilterOnNoPropertiesViewWarning(DMSSchemaWarning):
 @dataclass(frozen=True)
 class NodeTypeFilterOnParentViewWarning(DMSSchemaWarning):
     description = (
-        "Setting a node type filter on a parent view. This is no "
+        "Setting a node type filter on a parent view. This is not "
         "recommended as parent views are typically used for multiple type of nodes."
     )
     fix = "Use a HasData filter instead"
@@ -444,3 +462,73 @@ class NodeTypeFilterOnParentViewWarning(DMSSchemaWarning):
         output = super().dump()
         output["view_id"] = self.view_id.dump()
         return output
+
+
+@dataclass(frozen=True)
+class HasDataFilterOnViewWithReferencesWarning(DMSSchemaWarning):
+    description = (
+        "Setting a hasData filter on a solution view which reference other containers is not recommended."
+        "This will lead to no nodes being returned when querying the solution view."
+    )
+    fix = "Use a node type filter instead"
+    error_name: ClassVar[str] = "HasDataFilterOnReferencedViewWarning"
+
+    view_id: dm.ViewId
+    references: list[dm.ViewId]
+
+    def message(self) -> str:
+        return (
+            f"Setting a hasData filter on view {self.view_id} which references other views {self.references}. "
+            "This is not recommended as it will lead to no nodes being returned when querying the solution view."
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["view_id"] = self.view_id.dump()
+        output["references"] = [view.dump() for view in sorted(self.references, key=lambda x: x.as_tuple())]
+        return output
+
+
+@dataclass(frozen=True)
+class OtherDataModelsInSpaceWarning(DMSSchemaWarning):
+    description = "The space contains other data models"
+    fix = "Move the data models to their respective spaces."
+    error_name: ClassVar[str] = "OtherDataModelsInSpaceWarning"
+    space: str
+    data_models: list[dm.DataModelId]
+
+    def message(self) -> str:
+        return (
+            f"The space {self.space} contains data models from other spaces: {self.data_models}. {self.fix}"
+            " It is recommended to only have one data model per space. This avoid potential conflicts and "
+            "makes it easier to manage the data models."
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["space"] = self.space
+        output["data_models"] = [data_model.dump() for data_model in self.data_models]
+        return output
+
+
+@dataclass(frozen=True)
+class SolutionOnTopOfSolutionModelWarning(DMSSchemaWarning):
+    description = "The data model is a solution on top of another solution"
+    fix = "Use the base solution as the data model"
+    error_name: ClassVar[str] = "SolutionOnTopOfSolutionModelWarning"
+    data_model: dm.DataModelId
+    base_data_model: dm.DataModelId
+
+    def message(self) -> str:
+        return (
+            f"The data model {self.data_model} is a solution model on top of another solution {self.base_data_model} "
+            "model. This is not recommended as it can lead to confusion and errors. It is very hard to "
+            "maintain a nested structure of solution models. Instead, only build solution models on "
+            "top of enterprise models."
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["data_model"] = self.data_model.dump()
+        output["base_data_model"] = self.base_data_model.dump()
+        return output

cognite/neat/rules/issues/fileread.py

@@ -9,10 +9,13 @@ __all__ = [
     "InvalidFileFormatWarning",
     "UnsupportedSpecWarning",
     "UnknownItemWarning",
+    "FailedLoadWarning",
     "BugInImporterWarning",
     "FileReadError",
     "FileNotFoundError",
     "FileNotAFileError",
+    "InvalidFileFormatError",
+    "FailedStringLoadError",
 ]
 
 
@@ -96,6 +99,26 @@ class UnknownItemWarning(FileReadWarning):
         return output
 
 
+@dataclass(frozen=True)
+class FailedLoadWarning(FileReadWarning):
+    description = "The file content is invalid"
+    fix = "Check if the file content is valid"
+
+    expected_format: str
+    error_message: str
+
+    def message(self) -> str:
+        return (
+            f"Failed to load {self.filepath.name}. Expected format: {self.expected_format}. Error: {self.error_message}"
+        )
+
+    def dump(self) -> dict[str, str | None]:
+        output = super().dump()
+        output["expected_format"] = self.expected_format
+        output["error_message"] = self.error_message
+        return output
+
+
 @dataclass(frozen=True)
 class BugInImporterWarning(FileReadWarning):
     description = "A bug was raised during reading."
@@ -143,6 +166,24 @@ class InvalidFileFormatError(FileReadError):
         return f"{self.filepath} is not in the expected format. Expected format: {self.expected_format}."
 
 
+@dataclass(frozen=True)
+class FailedStringLoadError(NeatValidationError):
+    description = "The file content is invalid"
+    fix = "Check if the file content is valid"
+
+    expected_format: str
+    error_message: str
+
+    def message(self) -> str:
+        return f"Failed to load string. Expected format: {self.expected_format}. Error: {self.error_message}"
+
+    def dump(self) -> dict[str, str | None]:
+        output = super().dump()
+        output["expected_format"] = self.expected_format
+        output["error_message"] = self.error_message
+        return output
+
+
 @dataclass(frozen=True)
 class NoFilesFoundError(FileReadError):
     description = "No files were found in the directory"

cognite/neat/rules/issues/importing.py

@@ -10,11 +10,19 @@ __all__ = [
     "IgnoredComponentWarning",
     "UnknownPropertyWarning",
     "UnknownValueTypeWarning",
+    "MissingContainerWarning",
+    "MissingContainerPropertyWarning",
+    "MultipleDataModelsWarning",
+    "UnknownPropertyTypeWarning",
+    "FailedToInferValueTypeWarning",
+    "UnknownContainerConstraintWarning",
+    "NoDataModelError",
     "ModelImportError",
     "InvalidComponentError",
     "MissingParentDefinitionError",
     "MissingIdentifierError",
     "UnsupportedPropertyTypeError",
+    "APIError",
 ]
 
 
@@ -132,12 +140,159 @@ class UnknownValueTypeWarning(ModelImportWarning):
         )
 
 
+@dataclass(frozen=True)
+class MultipleDataModelsWarning(ModelImportWarning):
+    description = "Multiple data models detected. This is not supported."
+    fix = "Remove the extra data models."
+
+    data_models: list[str]
+
+    def message(self) -> str:
+        return f"Multiple data models detected: {self.data_models}. Will only import the first one."
+
+    def dump(self) -> dict[str, list[str]]:
+        return {"data_models": self.data_models}
+
+
+@dataclass(frozen=True)
+class MissingContainerWarning(ModelImportWarning):
+    description = "Missing container definition."
+    fix = "Add a container definition."
+
+    view_id: str
+    property_: str
+    container_id: str
+
+    def message(self) -> str:
+        return (
+            f"Container '{self.container_id}' is missing. "
+            f"Will skip property '{self.property_}' of view '{self.view_id}'."
+        )
+
+    def dump(self) -> dict[str, str]:
+        return {"view_id": self.view_id, "property": self.property_, "container_id": self.container_id}
+
+
+@dataclass(frozen=True)
+class MissingContainerPropertyWarning(ModelImportWarning):
+    description = "Missing container property definition."
+    fix = "Add a container property definition."
+
+    view_id: str
+    property_: str
+    container_id: str
+
+    def message(self) -> str:
+        return (
+            f"Container '{self.container_id}' is missing property '{self.property_}'. "
+            f"This property will be skipped for view '{self.view_id}'."
+        )
+
+    def dump(self) -> dict[str, str]:
+        return {"view_id": self.view_id, "property": self.property_, "container_id": self.container_id}
+
+
+@dataclass(frozen=True)
+class UnknownPropertyTypeWarning(ModelImportWarning):
+    description = "Unknown property type. This will be ignored in the imports."
+    fix = "Set to a supported property type."
+    view_id: str
+    property_id: str
+    property_type: str
+
+    def message(self) -> str:
+        return (
+            f"Unknown property type '{self.property_type}' for property '{self.property_id}' "
+            f"of view '{self.view_id}'. This will be ignored in the imports."
+        )
+
+    def dump(self) -> dict[str, str]:
+        return {"view_id": self.view_id, "property_id": self.property_id, "property_type": self.property_type}
+
+
+@dataclass(frozen=True)
+class UnknownContainerConstraintWarning(ModelImportWarning):
+    description = "Unknown container constraint. This will be ignored in the imports."
+    fix = "Set to a supported container constraint."
+    container_id: str
+    property_id: str
+    constraint: str
+
+    def message(self) -> str:
+        return (
+            f"Unknown container constraint '{self.constraint}' for property '{self.property_id}' of container "
+            f"'{self.container_id}'. This will be ignored in the imports."
+        )
+
+    def dump(self) -> dict[str, str]:
+        return {"container_id": self.container_id, "property_id": self.property_id, "constraint": self.constraint}
+
+
+@dataclass(frozen=True)
+class FailedToInferValueTypeWarning(ModelImportWarning):
+    description = "Failed to infer value type. This will be ignored in the imports."
+    fix = "Set to a supported value type."
+    view_id: str
+    property_id: str
+
+    def message(self) -> str:
+        return (
+            f"Failed to infer value type for property '{self.property_id}' of view '{self.view_id}'. "
+            f"This will be ignored in the imports."
+        )
+
+    def dump(self) -> dict[str, str]:
+        return {"view_id": self.view_id, "property_id": self.property_id}
+
+
+@dataclass(frozen=True)
+class APIWarning(ModelImportWarning):
+    description = "An error was raised."
+    fix = "No fix is available."
+
+    error_message: str
+
+    def message(self) -> str:
+        return self.error_message
+
+    def dump(self) -> dict[str, str]:
+        return {"error_message": self.error_message}
+
+
 @dataclass(frozen=True)
 class ModelImportError(NeatValidationError, ABC):
     description = "An error was raised during importing."
     fix = "No fix is available."
 
 
+@dataclass(frozen=True)
+class NoDataModelError(ModelImportError):
+    description = "No data model found.."
+    fix = "Check if the data model exists in the source."
+
+    error_message: str
+
+    def message(self) -> str:
+        return self.error_message
+
+    def dump(self) -> dict[str, str]:
+        return {"error_message": self.error_message}
+
+
+@dataclass(frozen=True)
+class APIError(ModelImportError):
+    description = "An error was raised during importing."
+    fix = "No fix is available."
+
+    error_message: str
+
+    def message(self) -> str:
+        return self.error_message
+
+    def dump(self) -> dict[str, str]:
+        return {"error_message": self.error_message}
+
+
 @dataclass(frozen=True)
 class InvalidComponentError(ModelImportError, ABC):
     description = "This is a base class for all errors related invalid component definitions"

cognite/neat/rules/issues/spreadsheet.py

@@ -176,15 +176,18 @@ class InvalidRowUnknownSheetError(InvalidRowError):
     ) -> Self:
         sheet_name, _, row, column, *__ = error["loc"]
         reader = (read_info_by_sheet or {}).get(str(sheet_name), SpreadsheetRead())
-        return cls(
-            column=str(column),
-            row=reader.adjusted_row_number(int(row)),
-            actual_sheet_name=str(sheet_name),
-            type=error["type"],
-            msg=error["msg"],
-            input=error.get("input"),
-            url=str(url) if (url := error.get("url")) else None,
-        )
+        try:
+            return cls(
+                column=str(column),
+                row=reader.adjusted_row_number(int(row)),
+                actual_sheet_name=str(sheet_name),
+                type=error["type"],
+                msg=error["msg"],
+                input=error.get("input"),
+                url=str(url) if (url := error.get("url")) else None,
+            )
+        except ValueError:
+            return DefaultPydanticError.from_pydantic_error(error)  # type: ignore[return-value]
 
     def dump(self) -> dict[str, Any]:
         output = super().dump()

cognite/neat/rules/models/entities.py

@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
 from functools import total_ordering
 from typing import Annotated, Any, ClassVar, Generic, TypeVar, cast
 
-from cognite.client.data_classes.data_modeling.ids import ContainerId, DataModelId, PropertyId, ViewId
+from cognite.client.data_classes.data_modeling.ids import ContainerId, DataModelId, NodeId, PropertyId, ViewId
 from pydantic import AnyHttpUrl, BaseModel, BeforeValidator, Field, PlainSerializer, model_serializer, model_validator
 
 if sys.version_info >= (3, 11):
@@ -30,6 +30,7 @@ class EntityTypes(StrEnum):
     data_value_type = "data_value_type"  # these are strings, floats, ...
     xsd_value_type = "xsd_value_type"
     dms_value_type = "dms_value_type"
+    dms_node = "dms_node"
     view = "view"
     reference_entity = "reference_entity"
     container = "container"
@@ -248,7 +249,7 @@ class UnknownEntity(ClassEntity):
         return str(Unknown)
 
 
-T_ID = TypeVar("T_ID", bound=ContainerId | ViewId | DataModelId | PropertyId | None)
+T_ID = TypeVar("T_ID", bound=ContainerId | ViewId | DataModelId | PropertyId | NodeId | None)
 
 
 class DMSEntity(Entity, Generic[T_ID], ABC):
@@ -302,7 +303,9 @@ class ContainerEntity(DMSEntity[ContainerId]):
 class DMSVersionedEntity(DMSEntity[T_ID], ABC):
     version: str
 
-    def as_class(self) -> ClassEntity:
+    def as_class(self, skip_version: bool = False) -> ClassEntity:
+        if skip_version:
+            return ClassEntity(prefix=self.space, suffix=self.external_id)
         return ClassEntity(prefix=self.space, suffix=self.external_id, version=self.version)
 
 
@@ -315,10 +318,13 @@ class ViewEntity(DMSVersionedEntity[ViewId]):
         return ViewId(space=self.space, external_id=self.external_id, version=self.version)
 
     @classmethod
-    def from_id(cls, id: ViewId) -> "ViewEntity":
-        if id.version is None:
+    def from_id(cls, id: ViewId, default_version: str | None = None) -> "ViewEntity":
+        if id.version is not None:
+            return cls(space=id.space, externalId=id.external_id, version=id.version)
+        elif default_version is not None:
+            return cls(space=id.space, externalId=id.external_id, version=default_version)
+        else:
             raise ValueError("Version must be specified")
-        return cls(space=id.space, externalId=id.external_id, version=id.version)
 
 
 class DMSUnknownEntity(DMSEntity[None]):
@@ -376,6 +382,17 @@ class DataModelEntity(DMSVersionedEntity[DataModelId]):
         return cls(space=id.space, externalId=id.external_id, version=id.version)
 
 
+class DMSNodeEntity(DMSEntity[NodeId]):
+    type_: ClassVar[EntityTypes] = EntityTypes.dms_node
+
+    def as_id(self) -> NodeId:
+        return NodeId(space=self.space, external_id=self.external_id)
+
+    @classmethod
+    def from_id(cls, id: NodeId) -> "DMSNodeEntity":
+        return cls(space=id.space, externalId=id.external_id)
+
+
 class ReferenceEntity(ClassEntity):
     type_: ClassVar[EntityTypes] = EntityTypes.reference_entity
     prefix: str
@@ -391,6 +408,12 @@ class ReferenceEntity(ClassEntity):
             raise ValueError("Property is not defined or prefix is not defined or suffix is unknown")
         return PropertyId(source=self.as_view_id(), property=self.property_)
 
+    def as_node_id(self) -> NodeId:
+        return NodeId(space=self.prefix, external_id=self.suffix)
+
+    def as_node_entity(self) -> DMSNodeEntity:
+        return DMSNodeEntity(space=self.prefix, externalId=self.suffix)
+
     def as_class_entity(self) -> ClassEntity:
         return ClassEntity(prefix=self.prefix, suffix=self.suffix, version=self.version)
 

cognite/neat/rules/models/rules/_base.py

@@ -27,8 +27,6 @@ from pydantic import (
 )
 from pydantic.fields import FieldInfo
 
-from cognite.neat.rules.models.entities import ClassEntity
-
 if sys.version_info >= (3, 11):
     from enum import StrEnum
     from typing import Self
@@ -140,6 +138,11 @@ class ExtensionCategory(StrEnum):
     rebuild = "rebuild"
 
 
+class DataModelType(StrEnum):
+    solution = "solution"
+    enterprise = "enterprise"
+
+
 class RoleTypes(StrEnum):
     domain_expert = "domain expert"
     information_architect = "information architect"
@@ -274,10 +277,6 @@ class BaseRules(RuleModel):
 
 # An sheet entity is either a class or a property.
 class SheetEntity(RuleModel):
-    class_: ClassEntity = Field(alias="Class")
-    name: str | None = Field(alias="Name", default=None)
-    description: str | None = Field(alias="Description", default=None)
-
     @field_validator("*", mode="before")
     def strip_string(cls, value: Any) -> Any:
         if isinstance(value, str):