cognite-neat 0.88.2__py3-none-any.whl → 0.89.0__py3-none-any.whl

cognite/neat/rules/models/dms/_schema.py

@@ -26,17 +26,20 @@ from cognite.client.data_classes.data_modeling.views import (
 from cognite.client.data_classes.transformations.common import Edges, EdgeType, Nodes, ViewInfo
 
 from cognite.neat.issues import NeatError
-from cognite.neat.issues.errors.properties import ReferredPropertyNotFoundError
-from cognite.neat.issues.errors.resources import ReferredResourceNotFoundError
-from cognite.neat.issues.neat_warnings.resources import FailedLoadingResourcesWarning, MultipleResourcesWarning
-from cognite.neat.rules import issues
-from cognite.neat.rules.issues.dms import (
-    ContainerPropertyUsedMultipleTimesError,
-    DirectRelationMissingSourceWarning,
-    DuplicatedViewInDataModelError,
-    IncompleteSchemaError,
-    MissingViewInModelWarning,
+from cognite.neat.issues.errors import (
+    NeatYamlError,
+    PropertyMappingDuplicatedError,
+    PropertyNotFoundError,
+    ResourceDuplicatedError,
+    ResourceNotFoundError,
 )
+from cognite.neat.issues.warnings import (
+    FileTypeUnexpectedWarning,
+    ResourceNotFoundWarning,
+    ResourceRetrievalWarning,
+    ResourcesDuplicatedWarning,
+)
+from cognite.neat.issues.warnings.user_modeling import DirectRelationMissingSourceWarning
 from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE
 from cognite.neat.utils.cdf.data_classes import (
     CogniteResourceDict,
@@ -93,11 +96,11 @@ class DMSSchema:
         directly_referenced_containers = view_by_id[view_id].referenced_containers()
         inherited_referenced_containers = set()
 
-        for view_id in view_inheritance:
-            if implemented_view := view_by_id.get(view_id):
+        for parent_id in view_inheritance:
+            if implemented_view := view_by_id.get(parent_id):
                 inherited_referenced_containers |= implemented_view.referenced_containers()
             else:
-                raise IncompleteSchemaError(missing_component=view_id).as_exception()
+                raise ResourceNotFoundError(parent_id, "view", view_id, "view")
 
         return directly_referenced_containers | inherited_referenced_containers
 
@@ -164,10 +167,14 @@ class DMSSchema:
             connection_referenced_view_ids |= cls._connection_references(view)
         connection_referenced_view_ids = connection_referenced_view_ids - existing_view_ids
         if connection_referenced_view_ids:
-            warnings.warn(MissingViewInModelWarning(data_model.as_id(), connection_referenced_view_ids), stacklevel=2)
+            for view_id in connection_referenced_view_ids:
+                warnings.warn(
+                    ResourceNotFoundWarning(view_id, "view", data_model_write.as_id(), "data model"),
+                    stacklevel=2,
+                )
             connection_referenced_views = view_loader.retrieve(list(connection_referenced_view_ids))
             if failed := connection_referenced_view_ids - set(connection_referenced_views.as_ids()):
-                warnings.warn(FailedLoadingResourcesWarning[dm.ViewId](frozenset(failed), "View"), stacklevel=2)
+                warnings.warn(ResourceRetrievalWarning(frozenset(failed), "view"), stacklevel=2)
             views.extend(connection_referenced_views)
 
         # We need to include parent views in the schema to make sure that the schema is valid.
@@ -262,10 +269,11 @@ class DMSSchema:
                     data.setdefault(attr_name, [])
                     context.setdefault(attr_name, [])
                     try:
-                        # Using CSafeLoader over safe_load for ~10x speedup
                         loaded = yaml.safe_load(yaml_file.read_text())
                     except Exception as e:
-                        warnings.warn(issues.fileread.InvalidFileFormatWarning(yaml_file, str(e)), stacklevel=2)
+                        warnings.warn(
+                            FileTypeUnexpectedWarning(yaml_file, frozenset([".yaml", ".yml"]), str(e)), stacklevel=2
+                        )
                         continue
 
                     if isinstance(loaded, list):
@@ -355,7 +363,9 @@ class DMSSchema:
                         try:
                             loaded = yaml.safe_load(zip_ref.read(file_info).decode())
                         except Exception as e:
-                            warnings.warn(issues.fileread.InvalidFileFormatWarning(filename, str(e)), stacklevel=2)
+                            warnings.warn(
+                                FileTypeUnexpectedWarning(filename, frozenset([".yaml", ".yml"]), str(e)), stacklevel=2
+                            )
                             continue
                         if isinstance(loaded, list):
                             data[attr_name].extend(loaded)
@@ -411,11 +421,9 @@ class DMSSchema:
             try:
                 data_dict = yaml.safe_load(data)
             except Exception as e:
-                raise issues.fileread.FailedStringLoadError(".yaml", str(e)).as_exception() from None
+                raise NeatYamlError(str(e)) from None
             if not isinstance(data_dict, dict) and all(isinstance(v, list) for v in data_dict.values()):
-                raise issues.fileread.FailedStringLoadError(
-                    "dict[str, list[Any]]", f"Invalid data structure: {type(data)}"
-                ).as_exception() from None
+                raise NeatYamlError(f"Invalid data structure: {type(data)}", "dict[str, list[Any]]") from None
         else:
             data_dict = data
         loaded: dict[str, Any] = {}
@@ -423,20 +431,32 @@ class DMSSchema:
             if items := data_dict.get(attr.name) or data_dict.get(to_camel(attr.name)):
                 if attr.name == "data_model":
                     if isinstance(items, list) and len(items) > 1:
-                        warnings.warn(
-                            MultipleResourcesWarning[str](
-                                frozenset([item.get("externalId", "Unknown") for item in items]),
-                                "DataModel",
-                            ),
-                            stacklevel=2,
-                        )
+                        try:
+                            data_model_ids = [dm.DataModelId.load(item) for item in items]
+                        except Exception as e:
+                            data_model_file = context.get(attr.name, [Path("UNKNOWN")])[0]
+                            warnings.warn(
+                                FileTypeUnexpectedWarning(
+                                    data_model_file, frozenset([dm.DataModelApply.__name__]), str(e)
+                                ),
+                                stacklevel=2,
+                            )
+                        else:
+                            warnings.warn(
+                                ResourcesDuplicatedWarning(
+                                    frozenset(data_model_ids),
+                                    "data model",
+                                    "Will use the first DataModel.",
+                                ),
+                                stacklevel=2,
+                            )
                     item = items[0] if isinstance(items, list) else items
                     try:
                         loaded[attr.name] = dm.DataModelApply.load(item)
                     except Exception as e:
                         data_model_file = context.get(attr.name, [Path("UNKNOWN")])[0]
                         warnings.warn(
-                            issues.fileread.FailedLoadWarning(data_model_file, dm.DataModelApply.__name__, str(e)),
+                            FileTypeUnexpectedWarning(data_model_file, frozenset([dm.DataModelApply.__name__]), str(e)),
                             stacklevel=2,
                         )
                 else:
@@ -453,7 +473,7 @@ class DMSSchema:
         resources = attr.type([])
         if not hasattr(attr.type, "_RESOURCE"):
             warnings.warn(
-                issues.fileread.FailedLoadWarning(Path("UNKNOWN"), attr.type.__name__, trigger_error), stacklevel=2
+                FileTypeUnexpectedWarning(Path("UNKNOWN"), frozenset([attr.type.__name__]), trigger_error), stacklevel=2
             )
             return resources
         # Fallback to load individual resources.
@@ -467,7 +487,9 @@ class DMSSchema:
                 except IndexError:
                     filepath = Path("UNKNOWN")
                 # We use repr(e) instead of str(e) to include the exception type in the warning message
-                warnings.warn(issues.fileread.FailedLoadWarning(filepath, single_cls.__name__, repr(e)), stacklevel=2)
+                warnings.warn(
+                    FileTypeUnexpectedWarning(filepath, frozenset([single_cls.__name__]), repr(e)), stacklevel=2
+                )
             else:
                 resources.append(loaded_instance)
         return resources
@@ -529,41 +551,31 @@ class DMSSchema:
         for container in self.containers.values():
             if container.space not in defined_spaces:
                 errors.add(
-                    ReferredResourceNotFoundError[str, dm.ContainerId](
-                        container.space, "Space", container.as_id(), "Container"
-                    )
+                    ResourceNotFoundError[str, dm.ContainerId](container.space, "space", container.as_id(), "container")
                 )
 
         for view in self.views.values():
             view_id = view.as_id()
             if view.space not in defined_spaces:
-                errors.add(ReferredResourceNotFoundError[str, dm.ViewId](view.space, "Space", view_id, "View"))
+                errors.add(ResourceNotFoundError(view.space, "space", view_id, "view"))
 
             for parent in view.implements or []:
                 if parent not in defined_views:
-                    errors.add(
-                        ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
-                            parent, "View", view_id, "View", property_name="implements"
-                        )
-                    )
+                    errors.add(PropertyNotFoundError(parent, "view", "implements", view_id, "view"))
 
             for prop_name, prop in (view.properties or {}).items():
                 if isinstance(prop, dm.MappedPropertyApply):
                     ref_container = defined_containers.get(prop.container)
                     if ref_container is None:
-                        errors.add(
-                            ReferredResourceNotFoundError[dm.ContainerId, dm.ViewId](
-                                prop.container, "Container", view_id, "View"
-                            )
-                        )
+                        errors.add(ResourceNotFoundError(prop.container, "container", view_id, "view"))
                     elif prop.container_property_identifier not in ref_container.properties:
                         errors.add(
-                            ReferredPropertyNotFoundError[dm.ContainerId, dm.ViewId](
+                            PropertyNotFoundError(
                                 prop.container,
-                                "Container",
+                                "container",
+                                prop.container_property_identifier,
                                 view_id,
-                                "View",
-                                property_name=prop.container_property_identifier,
+                                "view",
                             )
                         )
                     else:
@@ -571,26 +583,19 @@ class DMSSchema:
 
                         if isinstance(container_property.type, dm.DirectRelation) and prop.source is None:
                             warnings.warn(
-                                DirectRelationMissingSourceWarning(view_id=view_id, property=prop_name), stacklevel=2
+                                DirectRelationMissingSourceWarning(view_id, prop_name),
+                                stacklevel=2,
                             )
 
                 if isinstance(prop, dm.EdgeConnectionApply) and prop.source not in defined_views:
-                    errors.add(
-                        ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
-                            prop.source, "View", view_id, "View", property_name=prop_name
-                        )
-                    )
+                    errors.add(PropertyNotFoundError(prop.source, "view", prop_name, view_id, "view"))
 
                 if (
                     isinstance(prop, dm.EdgeConnectionApply)
                     and prop.edge_source is not None
                     and prop.edge_source not in defined_views
                 ):
-                    errors.add(
-                        ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
-                            prop.edge_source, "View", view_id, "View", property_name=prop_name
-                        )
-                    )
+                    errors.add(PropertyNotFoundError(prop.edge_source, "view", prop_name, view_id, "view"))
 
             # This allows for multiple view properties to be mapped to the same container property,
             # as long as they have different external_id, otherwise this will lead to raising
@@ -615,36 +620,36 @@ class DMSSchema:
                         == (container_id, container_property_identifier)
                     ]
                     errors.add(
-                        ContainerPropertyUsedMultipleTimesError(
-                            container=container_id,
-                            property=container_property_identifier,
-                            referred_by=frozenset({(view_id, prop_name) for prop_name in view_properties}),
+                        PropertyMappingDuplicatedError(
+                            container_id,
+                            "container",
+                            container_property_identifier,
+                            frozenset({dm.PropertyId(view_id, prop_name) for prop_name in view_properties}),
+                            "view property",
                         )
                     )
 
         if self.data_model:
             model = self.data_model
             if model.space not in defined_spaces:
-                errors.add(
-                    ReferredResourceNotFoundError[str, dm.DataModelId](
-                        model.space, "Space", model.as_id(), "Data Model"
-                    )
-                )
+                errors.add(ResourceNotFoundError(model.space, "space", model.as_id(), "data model"))
 
             view_counts: dict[dm.ViewId, int] = defaultdict(int)
             for view_id_or_class in model.views or []:
                 view_id = view_id_or_class if isinstance(view_id_or_class, dm.ViewId) else view_id_or_class.as_id()
                 if view_id not in defined_views:
-                    errors.add(
-                        ReferredResourceNotFoundError[dm.ViewId, dm.DataModelId](
-                            view_id, "View", model.as_id(), "DataModel"
-                        )
-                    )
+                    errors.add(ResourceNotFoundError(view_id, "view", model.as_id(), "data model"))
                 view_counts[view_id] += 1
 
             for view_id, count in view_counts.items():
                 if count > 1:
-                    errors.add(DuplicatedViewInDataModelError(referred_by=model.as_id(), view=view_id))
+                    errors.add(
+                        ResourceDuplicatedError(
+                            view_id,
+                            "view",
+                            repr(model.as_id()),
+                        )
+                    )
 
         return list(errors)
 
@@ -852,7 +857,9 @@ class PipelineSchema(DMSSchema):
                 try:
                     loaded = yaml.safe_load(yaml_file.read_text())
                 except Exception as e:
-                    warnings.warn(issues.fileread.InvalidFileFormatWarning(yaml_file, str(e)), stacklevel=2)
+                    warnings.warn(
+                        FileTypeUnexpectedWarning(yaml_file, frozenset([".yaml", ".yml"]), str(e)), stacklevel=2
+                    )
                     continue
                 if isinstance(loaded, list):
                     data[attr_name].extend(loaded)
@@ -918,7 +925,9 @@ class PipelineSchema(DMSSchema):
                         try:
                             loaded = yaml.safe_load(zip_ref.read(file_info).decode())
                         except Exception as e:
-                            warnings.warn(issues.fileread.InvalidFileFormatWarning(filepath, str(e)), stacklevel=2)
+                            warnings.warn(
+                                FileTypeUnexpectedWarning(filepath, frozenset([".yaml", ".yml"]), str(e)), stacklevel=2
+                            )
                             continue
                         if isinstance(loaded, list):
                             data[attr_name].extend(loaded)

cognite/neat/rules/models/dms/_serializer.py

@@ -1,7 +1,7 @@
 from typing import Any, ClassVar, cast
 
 from cognite.neat.rules.models import DMSRules
-from cognite.neat.rules.models.dms import DMSContainer, DMSProperty, DMSView
+from cognite.neat.rules.models.dms import DMSContainer, DMSEnum, DMSNode, DMSProperty, DMSView
 from cognite.neat.rules.models.entities import ReferenceEntity, ViewEntity
 
 
@@ -23,14 +23,19 @@ class _DMSRulesSerializer:
         self.view_name = "views"
         self.container_name = "containers"
         self.metadata_name = "metadata"
+        self.enum_name = "enum"
+        self.nodes_name = "nodes"
         self.prop_view = "view"
         self.prop_container = "container"
         self.prop_view_property = "view_property"
         self.prop_value_type = "value_type"
+        self.prop_connection = "connection"
         self.view_view = "view"
         self.view_implements = "implements"
         self.container_container = "container"
         self.container_constraint = "constraint"
+        self.nodes_node = "node"
+        self.enum_collection = "collection"
         self.reference = "Reference" if by_alias else "reference"
 
         if by_alias:
@@ -45,6 +50,7 @@ class _DMSRulesSerializer:
             self.prop_container = DMSProperty.model_fields[self.prop_container].alias or self.prop_container
             self.prop_view_property = DMSProperty.model_fields[self.prop_view_property].alias or self.prop_view_property
             self.prop_value_type = DMSProperty.model_fields[self.prop_value_type].alias or self.prop_value_type
+            self.prop_connection = DMSProperty.model_fields[self.prop_connection].alias or self.prop_connection
             self.view_view = DMSView.model_fields[self.view_view].alias or self.view_view
             self.view_implements = DMSView.model_fields[self.view_implements].alias or self.view_implements
             self.container_container = (
@@ -53,10 +59,15 @@ class _DMSRulesSerializer:
             self.container_constraint = (
                 DMSContainer.model_fields[self.container_constraint].alias or self.container_constraint
             )
+            self.nodes_node = DMSNode.model_fields[self.nodes_node].alias or self.nodes_node
+
             self.prop_name = DMSRules.model_fields[self.prop_name].alias or self.prop_name
             self.view_name = DMSRules.model_fields[self.view_name].alias or self.view_name
             self.container_name = DMSRules.model_fields[self.container_name].alias or self.container_name
             self.metadata_name = DMSRules.model_fields[self.metadata_name].alias or self.metadata_name
+            self.nodes_name = DMSRules.model_fields[self.nodes_name].alias or self.nodes_name
+            self.enum_name = DMSRules.model_fields[self.enum_name].alias or self.enum_name
+            self.enum_collection = DMSEnum.model_fields[self.enum_collection].alias or self.enum_collection
 
     def clean(self, dumped: dict[str, Any], as_reference: bool) -> dict[str, Any]:
         # Sorting to get a deterministic order
@@ -69,6 +80,16 @@ class _DMSRulesSerializer:
         else:
             dumped.pop(self.container_name, None)
 
+        if enum_data := dumped.get(self.enum_name):
+            dumped[self.enum_name] = sorted(enum_data["data"], key=lambda e: e[self.enum_collection])
+        else:
+            dumped.pop(self.enum_name, None)
+
+        if node_types_data := dumped.get(self.nodes_name):
+            dumped[self.nodes_name] = sorted(node_types_data["data"], key=lambda n: n[self.nodes_node])
+        else:
+            dumped.pop(self.nodes_name, None)
+
         for prop in dumped[self.prop_name]:
             if as_reference:
                 view_entity = cast(ViewEntity, ViewEntity.load(prop[self.prop_view]))
@@ -87,8 +108,25 @@ class _DMSRulesSerializer:
                     continue
                 if value := prop.get(field_name):
                     prop[field_name] = value.removeprefix(self.default_space).removesuffix(self.default_version_wrapped)
-            # Value type can have a property as well
-            prop[self.prop_value_type] = prop[self.prop_value_type].replace(self.default_version, "")
+            if isinstance(prop.get(self.prop_connection), str):
+                # Remove default values from connection (type, direction, properties)
+                default_type = f"type={self.default_space}{prop[self.view_view]}.{prop[self.prop_view_property]}"
+                default_type_space = f"type={self.default_space}"
+                default_properties = f"properties={self.default_space}"
+                default_direction = "direction=outwards"
+                prop[self.prop_connection] = (
+                    prop[self.prop_connection]
+                    .replace(self.default_version, "")
+                    .replace(default_type, "")
+                    .replace(default_type_space, "type=")
+                    .replace(default_properties, "properties=")
+                    .replace(default_direction, "")
+                    .replace("()", "")
+                    .replace("(,)", "")
+                    .replace("(,,)", "")
+                    .replace("(,", "(")
+                    .replace(",)", ")")
+                )
 
         for view in dumped[self.view_name]:
             if as_reference:
@@ -113,4 +151,7 @@ class _DMSRulesSerializer:
                 container[self.container_constraint] = ",".join(
                     constraint.strip().removeprefix(self.default_space) for constraint in value.split(",")
                 )
+
+        for node in dumped.get(self.nodes_name, []):
+            node[self.nodes_node] = node[self.nodes_node].removeprefix(self.default_space)
         return dumped