cognite-neat 1.0.9__py3-none-any.whl → 1.0.11__py3-none-any.whl

cognite/neat/_data_model/validation/dms/_connections.py

@@ -34,33 +34,27 @@ class ConnectionValueTypeUnexisting(DataModelValidator):
     issue_type = ConsistencyError
 
     def run(self) -> list[ConsistencyError]:
-        undefined_value_types = []
+        errors: list[ConsistencyError] = []
 
-        for (view, property_), value_type in self.local_resources.connection_end_node_types.items():
+        for (view, property_), value_type in self.validation_resources.connection_end_node_types.items():
             if value_type is None:
                 continue
 
-            if value_type in self.local_resources.views_by_reference:
-                continue
-
-            if (
-                self.modus_operandi == "additive" or value_type.space != self.local_resources.data_model_reference.space
-            ) and value_type in self.cdf_resources.views_by_reference:
+            if self.validation_resources.select_view(value_type) is not None:
                 continue
 
-            undefined_value_types.append((view, property_, value_type))
-
-        return [
-            ConsistencyError(
-                message=(
-                    f"View {view!s} connection {property_!s} has value type {value_type!s} "
-                    "which is not defined as a view in the data model neither exists in CDF."
-                ),
-                fix="Define necessary view",
-                code=self.code,
+            errors.append(
+                ConsistencyError(
+                    message=(
+                        f"View {view!s} connection {property_!s} has value type {value_type!s} "
+                        "which is not defined as a view in the data model neither exists in CDF."
+                    ),
+                    fix="Define necessary view",
+                    code=self.code,
+                )
             )
-            for (view, property_, value_type) in undefined_value_types
-        ]
+
+        return errors
 
 
 class ConnectionValueTypeUndefined(DataModelValidator):
@@ -84,23 +78,22 @@ class ConnectionValueTypeUndefined(DataModelValidator):
     issue_type = Recommendation
 
     def run(self) -> list[Recommendation]:
-        missing_value_types = []
+        recommendations: list[Recommendation] = []
 
-        for (view, property_), value_type in self.local_resources.connection_end_node_types.items():
+        for (view, property_), value_type in self.validation_resources.connection_end_node_types.items():
             if not value_type:
-                missing_value_types.append((view, property_))
-
-        return [
-            Recommendation(
-                message=(
-                    f"View {view!s} connection {property_!s} is missing value type (end node type)."
-                    " This yields ambiguous data model definition."
-                ),
-                fix="Define necessary value type",
-                code=self.code,
-            )
-            for (view, property_) in missing_value_types
-        ]
+                recommendations.append(
+                    Recommendation(
+                        message=(
+                            f"View {view!s} connection {property_!s} is missing value type (end node type)."
+                            " This yields ambiguous data model definition."
+                        ),
+                        fix="Define necessary value type",
+                        code=self.code,
+                    )
+                )
+
+        return recommendations
 
 
 @dataclass
@@ -159,9 +152,9 @@ class ReverseConnectionSourceViewMissing(DataModelValidator):
         for (target_view_ref, reverse_prop_name), (
             source_view_ref,
             through,
-        ) in self.local_resources.reverse_to_direct_mapping.items():
+        ) in self.validation_resources.reverse_to_direct_mapping.items():
             through = _normalize_through_reference(source_view_ref, through)
-            source_view = self._select_view_with_property(source_view_ref, through.identifier)
+            source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
 
             if not source_view:
                 errors.append(
@@ -204,14 +197,16 @@ class ReverseConnectionSourcePropertyMissing(DataModelValidator):
         for (target_view_ref, reverse_prop_name), (
             source_view_ref,
             through,
-        ) in self.local_resources.reverse_to_direct_mapping.items():
+        ) in self.validation_resources.reverse_to_direct_mapping.items():
             through = _normalize_through_reference(source_view_ref, through)
-            source_view = self._select_view_with_property(source_view_ref, through.identifier)
+            source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
 
             if not source_view:
                 continue  # Handled by ReverseConnectionSourceViewMissing
 
-            if not source_view.properties or through.identifier not in source_view.properties:
+            properties = self.validation_resources.properties_by_view.get(source_view_ref)
+            # we need expanded properties here
+            if not properties or through.identifier not in properties:
                 errors.append(
                     ConsistencyError(
                         message=(
@@ -252,17 +247,20 @@ class ReverseConnectionSourcePropertyWrongType(DataModelValidator):
         for (target_view_ref, reverse_prop_name), (
             source_view_ref,
             through,
-        ) in self.local_resources.reverse_to_direct_mapping.items():
+        ) in self.validation_resources.reverse_to_direct_mapping.items():
             through = _normalize_through_reference(source_view_ref, through)
-            source_view = self._select_view_with_property(source_view_ref, through.identifier)
+            source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
 
-            if not source_view or not source_view.properties:
-                continue  # Handled by other validators
+            if not source_view:
+                continue  # Handled by ReverseConnectionSourceViewMissing
 
-            source_property = source_view.properties.get(through.identifier)
-            if not source_property:
+            properties = self.validation_resources.properties_by_view.get(source_view_ref)
+            # we need expanded properties here
+            if not properties or through.identifier not in properties:
                 continue  # Handled by ReverseConnectionSourcePropertyMissing
 
+            source_property = properties[through.identifier]
+
             if not isinstance(source_property, ViewCorePropertyRequest):
                 errors.append(
                     ConsistencyError(
@@ -303,21 +301,27 @@ class ReverseConnectionContainerMissing(DataModelValidator):
         for (target_view_ref, reverse_prop_name), (
             source_view_ref,
             through,
-        ) in self.local_resources.reverse_to_direct_mapping.items():
+        ) in self.validation_resources.reverse_to_direct_mapping.items():
             through = _normalize_through_reference(source_view_ref, through)
-            source_view = self._select_view_with_property(source_view_ref, through.identifier)
+            source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
 
-            if not source_view or not source_view.properties:
-                continue  # Handled by other validators
+            if not source_view:
+                continue  # Handled by ReverseConnectionSourceViewMissing
 
-            source_property = source_view.properties.get(through.identifier)
-            if not source_property or not isinstance(source_property, ViewCorePropertyRequest):
-                continue  # Handled by other validators
+            properties = self.validation_resources.properties_by_view.get(source_view_ref)
+            # we need expanded properties here
+            if not properties or through.identifier not in properties:
+                continue  # Handled by ReverseConnectionSourcePropertyMissing
+
+            source_property = properties[through.identifier]
+
+            if not isinstance(source_property, ViewCorePropertyRequest):
+                continue  # Handled by ReverseConnectionSourcePropertyWrongType
 
             container_ref = source_property.container
             container_property_id = source_property.container_property_identifier
 
-            source_container = self._select_container_with_property(container_ref, container_property_id)
+            source_container = self.validation_resources.select_container(container_ref, container_property_id)
             if not source_container:
                 errors.append(
                     ConsistencyError(
@@ -361,21 +365,27 @@ class ReverseConnectionContainerPropertyMissing(DataModelValidator):
         for (target_view_ref, reverse_prop_name), (
             source_view_ref,
             through,
-        ) in self.local_resources.reverse_to_direct_mapping.items():
+        ) in self.validation_resources.reverse_to_direct_mapping.items():
             through = _normalize_through_reference(source_view_ref, through)
-            source_view = self._select_view_with_property(source_view_ref, through.identifier)
+            source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
 
-            if not source_view or not source_view.properties:
-                continue  # Handled by other validators
+            if not source_view:
+                continue  # Handled by ReverseConnectionSourceViewMissing
 
-            source_property = source_view.properties.get(through.identifier)
-            if not source_property or not isinstance(source_property, ViewCorePropertyRequest):
-                continue  # Handled by other validators
+            properties = self.validation_resources.properties_by_view.get(source_view_ref)
+            # we need expanded properties here
+            if not properties or through.identifier not in properties:
+                continue  # Handled by ReverseConnectionSourcePropertyMissing
+
+            source_property = properties[through.identifier]
+
+            if not isinstance(source_property, ViewCorePropertyRequest):
+                continue  # Handled by ReverseConnectionSourcePropertyWrongType
 
             container_ref = source_property.container
             container_property_id = source_property.container_property_identifier
 
-            source_container = self._select_container_with_property(container_ref, container_property_id)
+            source_container = self.validation_resources.select_container(container_ref, container_property_id)
             if not source_container:
                 continue  # Handled by ReverseConnectionContainerMissing
 
@@ -422,21 +432,27 @@ class ReverseConnectionContainerPropertyWrongType(DataModelValidator):
         for (target_view_ref, reverse_prop_name), (
             source_view_ref,
             through,
-        ) in self.local_resources.reverse_to_direct_mapping.items():
+        ) in self.validation_resources.reverse_to_direct_mapping.items():
             through = _normalize_through_reference(source_view_ref, through)
-            source_view = self._select_view_with_property(source_view_ref, through.identifier)
+            source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
 
-            if not source_view or not source_view.properties:
-                continue  # Handled by other validators
+            if not source_view:
+                continue  # Handled by ReverseConnectionSourceViewMissing
 
-            source_property = source_view.properties.get(through.identifier)
-            if not source_property or not isinstance(source_property, ViewCorePropertyRequest):
-                continue  # Handled by other validators
+            properties = self.validation_resources.properties_by_view.get(source_view_ref)
+            # we need expanded properties here
+            if not properties or through.identifier not in properties:
+                continue  # Handled by ReverseConnectionSourcePropertyMissing
+
+            source_property = properties[through.identifier]
+
+            if not isinstance(source_property, ViewCorePropertyRequest):
+                continue  # Handled by ReverseConnectionSourcePropertyWrongType
 
             container_ref = source_property.container
             container_property_id = source_property.container_property_identifier
 
-            source_container = self._select_container_with_property(container_ref, container_property_id)
+            source_container = self.validation_resources.select_container(container_ref, container_property_id)
             if not source_container or not source_container.properties:
                 continue  # Handled by other validators
 
@@ -485,16 +501,22 @@ class ReverseConnectionTargetMissing(DataModelValidator):
         for (target_view_ref, reverse_prop_name), (
             source_view_ref,
             through,
-        ) in self.local_resources.reverse_to_direct_mapping.items():
+        ) in self.validation_resources.reverse_to_direct_mapping.items():
             through = _normalize_through_reference(source_view_ref, through)
-            source_view = self._select_view_with_property(source_view_ref, through.identifier)
+            source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
 
-            if not source_view or not source_view.properties:
-                continue  # Handled by other validators
+            if not source_view:
+                continue  # Handled by ReverseConnectionSourceViewMissing
 
-            source_property = source_view.properties.get(through.identifier)
-            if not source_property or not isinstance(source_property, ViewCorePropertyRequest):
-                continue  # Handled by other validators
+            properties = self.validation_resources.properties_by_view.get(source_view_ref)
+            # we need expanded properties here
+            if not properties or through.identifier not in properties:
+                continue  # Handled by ReverseConnectionSourcePropertyMissing
+
+            source_property = properties[through.identifier]
+
+            if not isinstance(source_property, ViewCorePropertyRequest):
+                continue  # Handled by ReverseConnectionSourcePropertyWrongType
 
             actual_target_view = source_property.source
 
@@ -542,15 +564,21 @@ class ReverseConnectionPointsToAncestor(DataModelValidator):
         for (target_view_ref, reverse_prop_name), (
             source_view_ref,
             through,
-        ) in self.local_resources.reverse_to_direct_mapping.items():
+        ) in self.validation_resources.reverse_to_direct_mapping.items():
             through = _normalize_through_reference(source_view_ref, through)
-            source_view = self._select_view_with_property(source_view_ref, through.identifier)
+            source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
 
-            if not source_view or not source_view.properties:
-                continue  # Handled by other validators
+            if not source_view:
+                continue  # Handled by ReverseConnectionSourceViewMissing
+
+            properties = self.validation_resources.properties_by_view.get(source_view_ref)
+            # we need expanded properties here
+            if not properties or through.identifier not in properties:
+                continue  # Handled by ReverseConnectionSourcePropertyMissing
+
+            source_property = properties[through.identifier]
 
-            source_property = source_view.properties.get(through.identifier)
-            if not source_property or not isinstance(source_property, ViewCorePropertyRequest):
+            if not isinstance(source_property, ViewCorePropertyRequest):
                 continue  # Handled by other validators
 
             actual_target_view = source_property.source
@@ -558,7 +586,7 @@ class ReverseConnectionPointsToAncestor(DataModelValidator):
             if not actual_target_view:
                 continue  # Handled by ReverseConnectionTargetMissing
 
-            if self._is_ancestor_of_target(actual_target_view, target_view_ref):
+            if self.validation_resources.is_ancestor(target_view_ref, actual_target_view):
                 recommendations.append(
                     Recommendation(
                         message=(
@@ -602,15 +630,21 @@ class ReverseConnectionTargetMismatch(DataModelValidator):
         for (target_view_ref, reverse_prop_name), (
             source_view_ref,
             through,
-        ) in self.local_resources.reverse_to_direct_mapping.items():
+        ) in self.validation_resources.reverse_to_direct_mapping.items():
             through = _normalize_through_reference(source_view_ref, through)
-            source_view = self._select_view_with_property(source_view_ref, through.identifier)
+            source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
 
-            if not source_view or not source_view.properties:
-                continue  # Handled by other validators
+            if not source_view:
+                continue  # Handled by ReverseConnectionSourceViewMissing
 
-            source_property = source_view.properties.get(through.identifier)
-            if not source_property or not isinstance(source_property, ViewCorePropertyRequest):
+            properties = self.validation_resources.properties_by_view.get(source_view_ref)
+            # we need expanded properties here
+            if not properties or through.identifier not in properties:
+                continue  # Handled by ReverseConnectionSourcePropertyMissing
+
+            source_property = properties[through.identifier]
+
+            if not isinstance(source_property, ViewCorePropertyRequest):
                 continue  # Handled by other validators
 
             actual_target_view = source_property.source
@@ -618,7 +652,7 @@ class ReverseConnectionTargetMismatch(DataModelValidator):
             if not actual_target_view:
                 continue  # Handled by ReverseConnectionTargetMissing
 
-            if self._is_ancestor_of_target(actual_target_view, target_view_ref):
+            if self.validation_resources.is_ancestor(target_view_ref, actual_target_view):
                 continue  # Handled by ReverseConnectionTargetAncestor
 
             if actual_target_view != target_view_ref:

cognite/neat/_data_model/validation/dms/_consistency.py

@@ -28,22 +28,23 @@ class ViewSpaceVersionInconsistentWithDataModel(DataModelValidator):
     def run(self) -> list[Recommendation]:
         recommendations: list[Recommendation] = []
 
-        for view_ref in self.local_resources.views_by_reference:
+        if not self.validation_resources.merged_data_model.views:
+            return recommendations
+
+        space = self.validation_resources.merged_data_model.space
+        version = self.validation_resources.merged_data_model.version
+
+        for view_ref in self.validation_resources.merged_data_model.views:
             issue_description = ""
 
             if view_ref.space not in COGNITE_SPACES:
-                # notify about inconsisten space
-                if view_ref.space != self.local_resources.data_model_reference.space:
-                    issue_description = (
-                        f"space (view: {view_ref.space}, data model: {self.local_resources.data_model_reference.space})"
-                    )
+                # notify about inconsistent space
+                if view_ref.space != space:
+                    issue_description = f"space (view: {view_ref.space}, data model: {space})"
 
                 # or version if spaces are same
-                elif view_ref.version != self.local_resources.data_model_reference.version:
-                    issue_description = (
-                        f"version (view: {view_ref.version}, "
-                        f"data model: {self.local_resources.data_model_reference.version})"
-                    )
+                elif view_ref.version != version:
+                    issue_description = f"version (view: {view_ref.version}, data model: {version})"
 
             if issue_description:
                 recommendations.append(

cognite/neat/_data_model/validation/dms/_containers.py

@@ -36,16 +36,30 @@ class ExternalContainerDoesNotExist(DataModelValidator):
     def run(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
 
-        for view_ref, view in self.merged_views.items():
+        if not self.validation_resources.merged_data_model.views:
+            return errors
+
+        for view_ref in self.validation_resources.merged_data_model.views:
+            view = self.validation_resources.select_view(view_ref)
+
+            if not view:
+                raise RuntimeError(
+                    f"ImplementedViewNotExisting.run: View {view_ref!s} "
+                    "not found in local resources. This is a bug in NEAT."
+                )
+
+            if view.properties is None:
+                continue
+
             for property_ref, property_ in view.properties.items():
                 if not isinstance(property_, ViewCorePropertyRequest):
                     continue
 
-                if property_.container.space == self.local_resources.data_model_reference.space:
+                if property_.container.space == self.validation_resources.merged_data_model.space:
                     continue
 
                 # Check existence of container in CDF
-                if property_.container not in self.cdf_resources.containers_by_reference:
+                if property_.container not in self.validation_resources.cdf.containers:
                     errors.append(
                         ConsistencyError(
                             message=(
@@ -87,35 +101,47 @@ class ExternalContainerPropertyDoesNotExist(DataModelValidator):
     def run(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
 
-        for view_ref, view in self.merged_views.items():
-            for property_ref, property_ in view.properties.items():
-                if not isinstance(property_, ViewCorePropertyRequest):
-                    continue
+        if self.validation_resources.merged_data_model.views:
+            for view_ref in self.validation_resources.merged_data_model.views:
+                view = self.validation_resources.select_view(view_ref)
 
-                if property_.container.space == self.local_resources.data_model_reference.space:
-                    continue
+                if not view:
+                    raise RuntimeError(
+                        f"ImplementedViewNotExisting.run: View {view_ref!s} "
+                        "not found in local resources. This is a bug in NEAT."
+                    )
 
-                # Only check property if container exists in CDF
-                # this check is done in ExternalContainerDoesNotExist
-                if property_.container not in self.cdf_resources.containers_by_reference:
+                if view.properties is None:
                     continue
 
-                # Check existence of container property in CDF
-                if (
-                    property_.container_property_identifier
-                    not in self.cdf_resources.containers_by_reference[property_.container].properties
-                ):
-                    errors.append(
-                        ConsistencyError(
-                            message=(
-                                f"View {view_ref!s} property {property_ref!s} maps to "
-                                f"external container {property_.container!s} which does not have "
-                                f"property '{property_.container_property_identifier}' in CDF."
-                            ),
-                            fix="Define necessary container property in CDF",
-                            code=self.code,
+                for property_ref, property_ in view.properties.items():
+                    if not isinstance(property_, ViewCorePropertyRequest):
+                        continue
+
+                    if property_.container.space == self.validation_resources.merged_data_model.space:
+                        continue
+
+                    # Only check property if container exists in CDF
+                    # this check is done in ExternalContainerDoesNotExist
+                    if property_.container not in self.validation_resources.cdf.containers:
+                        continue
+
+                    # Check existence of container property in CDF
+                    if (
+                        property_.container_property_identifier
+                        not in self.validation_resources.cdf.containers[property_.container].properties
+                    ):
+                        errors.append(
+                            ConsistencyError(
+                                message=(
+                                    f"View {view_ref!s} property {property_ref!s} maps to "
+                                    f"external container {property_.container!s} which does not have "
+                                    f"property '{property_.container_property_identifier}' in CDF."
+                                ),
+                                fix="Define necessary container property in CDF",
+                                code=self.code,
+                            )
                         )
-                    )
 
         return errors
 
@@ -144,26 +170,19 @@ class RequiredContainerDoesNotExist(DataModelValidator):
     def run(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
 
-        for container_ref, container in self.local_resources.containers_by_reference.items():
+        for container_ref, container in self.validation_resources.merged.containers.items():
             if not container.constraints:
                 continue
 
-            for external_id, constraint in cast(dict[str, Constraint], container.constraints).items():
+            for constraint_ref, constraint in cast(dict[str, Constraint], container.constraints).items():
                 if not isinstance(constraint, RequiresConstraintDefinition):
                     continue
 
-                is_local = constraint.require.space == self.local_resources.data_model_reference.space
-                container_exists = (
-                    constraint.require in self.merged_containers
-                    if is_local
-                    else constraint.require in self.cdf_resources.containers_by_reference
-                )
-
-                if not container_exists:
+                if not self.validation_resources.select_container(constraint.require):
                     errors.append(
                         ConsistencyError(
                             message=(
-                                f"Container '{container_ref!s}' constraint '{external_id}' requires container "
+                                f"Container '{container_ref!s}' constraint '{constraint_ref}' requires container "
                                 f"'{constraint.require!s}' which does not exist."
                             ),
                             fix="Define necessary container in the data model",