cognite-neat 0.123.26__py3-none-any.whl → 1.0.22__py3-none-any.whl

cognite/neat/_data_model/validation/dms/_consistency.py

@@ -0,0 +1,58 @@
+"""Validators checking for consistency issues in data model."""
+
+from cognite.neat._data_model._constants import COGNITE_SPACES
+from cognite.neat._data_model.validation.dms._base import DataModelValidator
+from cognite.neat._issues import Recommendation
+
+BASE_CODE = "NEAT-DMS-CONSISTENCY"
+
+
+class ViewSpaceVersionInconsistentWithDataModel(DataModelValidator):
+    """Validates that views have consistent space and version with the data model.
+
+    ## What it does
+    Validates that all views in the data model have the same space and version as the data model.
+
+    ## Why is this bad?
+    If views have different space or version than the data model, it may lead to more demanding development and
+    maintenance efforts. The industry best practice is to keep views in the same space and version as the data model.
+
+    ## Example
+    If the data model is defined in space "my_space" version "v1", but a view is defined in the same spave but with
+    version "v2", this requires additional attention during deployment and maintenance of the data model.
+    """
+
+    code = f"{BASE_CODE}-001"
+    issue_type = Recommendation
+
+    def run(self) -> list[Recommendation]:
+        recommendations: list[Recommendation] = []
+
+        if not self.validation_resources.merged_data_model.views:
+            return recommendations
+
+        space = self.validation_resources.merged_data_model.space
+        version = self.validation_resources.merged_data_model.version
+
+        for view_ref in self.validation_resources.merged_data_model.views:
+            issue_description = ""
+
+            if view_ref.space not in COGNITE_SPACES:
+                # notify about inconsistent space
+                if view_ref.space != space:
+                    issue_description = f"space (view: {view_ref.space}, data model: {space})"
+
+                # or version if spaces are same
+                elif view_ref.version != version:
+                    issue_description = f"version (view: {view_ref.version}, data model: {version})"
+
+            if issue_description:
+                recommendations.append(
+                    Recommendation(
+                        message=(f"View {view_ref!s} has inconsistent {issue_description} with the data model."),
+                        fix="Update view version and/or space to match data model",
+                        code=self.code,
+                    )
+                )
+
+        return recommendations

cognite/neat/_data_model/validation/dms/_containers.py

@@ -0,0 +1,199 @@
+"""Validators for checking containers in the data model."""
+
+from pyparsing import cast
+
+from cognite.neat._data_model.models.dms._constraints import Constraint, RequiresConstraintDefinition
+from cognite.neat._data_model.models.dms._view_property import ViewCorePropertyRequest
+from cognite.neat._data_model.validation.dms._base import DataModelValidator
+from cognite.neat._issues import ConsistencyError
+
+BASE_CODE = "NEAT-DMS-CONTAINER"
+
+
+class ExternalContainerDoesNotExist(DataModelValidator):
+    """
+    Validates that any container referenced by a view property, when the
+    referenced container does not belong to the data model's space, exists in CDF.
+
+    ## What it does
+    For each view property that maps to a container in a different space than the data model,
+    this validator checks that the referenced external container exists in CDF.
+
+    ## Why is this bad?
+    If a view property references a container that does not exist in CDF,
+    the data model cannot be deployed. The affected view property will not function, and the
+    deployment of the entire data model will fail.
+
+    ## Example
+    View `my_space:WindTurbine` has a property `location` that maps to container
+    `other_space:WindTurbineContainer`, where `other_space` differs from `my_space`. If that
+    container does not exist in CDF, the model cannot be deployed.
+    """
+
+    code = f"{BASE_CODE}-001"
+    issue_type = ConsistencyError
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+
+        if not self.validation_resources.merged_data_model.views:
+            return errors
+
+        for view_ref in self.validation_resources.merged_data_model.views:
+            view = self.validation_resources.select_view(view_ref)
+
+            if not view:
+                raise RuntimeError(
+                    f"{type(self).__name__}: View {view_ref!s} not found in local resources. This is a bug in NEAT."
+                )
+
+            if view.properties is None:
+                continue
+
+            for property_ref, property_ in view.properties.items():
+                if not isinstance(property_, ViewCorePropertyRequest):
+                    continue
+
+                if property_.container.space == self.validation_resources.merged_data_model.space:
+                    continue
+
+                # Check existence of container in CDF
+                if property_.container not in self.validation_resources.cdf.containers:
+                    errors.append(
+                        ConsistencyError(
+                            message=(
+                                f"View {view_ref!s} property {property_ref!s} maps to "
+                                f"external container {property_.container!s} which does not exist in CDF."
+                            ),
+                            fix="Define necessary container in CDF",
+                            code=self.code,
+                        )
+                    )
+
+        return errors
+
+
+class ExternalContainerPropertyDoesNotExist(DataModelValidator):
+    """
+    Validates that any container property referenced by a view property, when the
+    referenced container does not belong to the data model's space, exists in CDF.
+
+    ## What it does
+    For each view property that maps to a container in a different space than the data model,
+    this validator checks that the referenced container property exists in that external container in CDF.
+    This validator only runs if the external container exists in CDF.
+
+    ## Why is this bad?
+    If a view property references a container property that does not exist in CDF,
+    the data model cannot be deployed. The affected view property will not function, and the
+    deployment of the entire data model will fail.
+
+    ## Example
+    View `my_space:WindTurbine` has a property `location` that maps to container property
+    `gpsCoordinates` in `other_space:WindTurbineContainer`. If `gpsCoordinates` does not exist
+    in that container in CDF, deployment will fail.
+    """
+
+    code = f"{BASE_CODE}-002"
+    issue_type = ConsistencyError
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+
+        if self.validation_resources.merged_data_model.views:
+            for view_ref in self.validation_resources.merged_data_model.views:
+                view = self.validation_resources.select_view(view_ref)
+
+                if not view:
+                    raise RuntimeError(
+                        f"{type(self).__name__}: View {view_ref!s} not found in local resources. This is a bug in NEAT."
+                    )
+
+                if view.properties is None:
+                    continue
+
+                for property_ref, property_ in view.properties.items():
+                    if not isinstance(property_, ViewCorePropertyRequest):
+                        continue
+
+                    if property_.container.space == self.validation_resources.merged_data_model.space:
+                        continue
+
+                    # Only check property if container exists in CDF
+                    # this check is done in ExternalContainerDoesNotExist
+                    if property_.container not in self.validation_resources.cdf.containers:
+                        continue
+
+                    # Check existence of container property in CDF
+                    if (
+                        property_.container_property_identifier
+                        not in self.validation_resources.cdf.containers[property_.container].properties
+                    ):
+                        errors.append(
+                            ConsistencyError(
+                                message=(
+                                    f"View {view_ref!s} property {property_ref!s} maps to "
+                                    f"external container {property_.container!s} which does not have "
+                                    f"property '{property_.container_property_identifier}' in CDF."
+                                ),
+                                fix="Define necessary container property in CDF",
+                                code=self.code,
+                            )
+                        )
+
+        return errors
+
+
+class RequiredContainerDoesNotExist(DataModelValidator):
+    """
+    Validates that any container required by another container exists in the data model.
+
+    ## What it does
+    For each container in the data model, this validator checks that any container it
+    requires (via requires constraints) exists either in the data model or in CDF.
+
+    ## Why is this bad?
+    If a container requires another container that does not exist in the data model or in CDF,
+    the data model cannot be deployed. The affected container will not function, and
+    the deployment of the entire data model will fail.
+
+    ## Example
+    Container `windy_space:WindTurbineContainer` has a constraint requiring `windy_space:LocationContainer`.
+    If `windy_space:LocationContainer` does not exist in the data model or in CDF, deployment will fail.
+    """
+
+    code = f"{BASE_CODE}-003"
+    issue_type = ConsistencyError
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+
+        for container_ref in self.validation_resources.merged.containers:
+            container = self.validation_resources.select_container(container_ref)
+
+            if not container:
+                raise RuntimeError(
+                    f"{type(self).__name__}: Container {container_ref!s} "
+                    "not found in local resources. This is a bug in NEAT."
+                )
+
+            if not container.constraints:
+                continue
+
+            for constraint_ref, constraint in cast(dict[str, Constraint], container.constraints).items():
+                if not isinstance(constraint, RequiresConstraintDefinition):
+                    continue
+
+                if not self.validation_resources.select_container(constraint.require):
+                    errors.append(
+                        ConsistencyError(
+                            message=(
+                                f"Container '{container_ref!s}' constraint '{constraint_ref}' requires container "
+                                f"'{constraint.require!s}' which does not exist."
+                            ),
+                            fix="Define necessary container in the data model",
+                            code=self.code,
+                        )
+                    )
+
+        return errors

cognite/neat/_data_model/validation/dms/_limits.py

@@ -0,0 +1,368 @@
+"""Validators for checking if defined data model is within CDF DMS schema limits."""
+
+from typing import Literal
+
+from cognite.neat._data_model.models.dms._container import ContainerRequest
+from cognite.neat._data_model.models.dms._data_types import EnumProperty, ListablePropertyTypeDefinition
+from cognite.neat._data_model.models.dms._indexes import BtreeIndex, InvertedIndex
+from cognite.neat._data_model.models.dms._view_property import (
+    ViewCorePropertyRequest,
+)
+from cognite.neat._data_model.validation.dms._base import (
+    DataModelValidator,
+)
+from cognite.neat._issues import ConsistencyError
+
+BASE_CODE = "NEAT-DMS-LIMITS"
+
+
+class DataModelViewCountIsOutOfLimits(DataModelValidator):
+    """Validates that the data model does not exceed the maximum number of views.
+
+    ## What it does
+    This validator checks that the total number of views referenced by the data model
+    does not exceed the limit defined in the CDF project.
+
+    ## Why is this bad?
+    CDF enforces limits on the number of views per data model to ensure optimal performance
+    and resource utilization.
+
+    ## Example
+    If the CDF project has a limit of 100 views per data model, and the data model
+    references 120 views, this validator will raise a ConsistencyError issue.
+
+    """
+
+    code = f"{BASE_CODE}-DATA-MODEL-001"
+    issue_type = ConsistencyError
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+
+        if self.validation_resources.merged_data_model.views is None:
+            errors.append(
+                ConsistencyError(
+                    message="The data model does not have any views. This means it is not a data model.",
+                    code=self.code,
+                )
+            )
+
+        elif (
+            len(self.validation_resources.merged_data_model.views) > self.validation_resources.limits.data_models.views
+        ):
+            errors.append(
+                ConsistencyError(
+                    message=(
+                        f"The data model references {len(self.validation_resources.merged_data_model.views)} views, "
+                        "which exceeds the limit of "
+                        f"{self.validation_resources.limits.data_models.views} views per data model."
+                    ),
+                    code=self.code,
+                )
+            )
+        return errors
+
+
+### View level limits
+
+
+class ViewPropertyCountIsOutOfLimits(DataModelValidator):
+    """Validates that a view does not exceed the maximum number of properties.
+
+    ## What it does
+    Checks that the view has no more properties than the CDF limit allows.
+
+    ## Why is this bad?
+    CDF enforces limits on the number of properties per view to ensure optimal performance.
+
+    ## Example
+    If a view has 150 properties and the CDF limit is 100 properties per view,
+    this validator will raise a ConsistencyError issue.
+    """
+
+    code = f"{BASE_CODE}-VIEW-001"
+    issue_type = ConsistencyError
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+
+        for view_ref in self.validation_resources.merged_data_model.views or []:
+            if not (view := self.validation_resources.expand_view_properties(view_ref)):
+                raise RuntimeError(f"{type(self).__name__}: View {view_ref!s} not found. This is a bug in NEAT.")
+
+            if view.properties and len(view.properties) > self.validation_resources.limits.views.properties:
+                errors.append(
+                    ConsistencyError(
+                        message=(
+                            f"View {view_ref!s} has {len(view.properties)} properties,"
+                            " which exceeds the limit of "
+                            f"{self.validation_resources.limits.views.properties} properties per view."
+                        ),
+                        code=self.code,
+                    )
+                )
+
+            elif not view.properties:
+                errors.append(
+                    ConsistencyError(
+                        message=(
+                            f"View {view_ref!s} does "
+                            "not have any properties defined, either directly or through implements."
+                        ),
+                        fix="Define at least one property for view",
+                        code=self.code,
+                    )
+                )
+
+        return errors
+
+
+class ViewContainerCountIsOutOfLimits(DataModelValidator):
+    """Validates that a view does not reference too many containers.
+
+    ## What it does
+    Checks that the view references no more containers than the CDF limit allows.
+
+    ## Why is this bad?
+    CDF enforces limits on the number of containers per view to prevent overly complex view definitions, leading
+    to too many joins and performance degradation.
+
+    ## Example
+    If a view references 20 containers and the CDF limit is 10 containers per view,
+    this validator will raise a ConsistencyError issue.
+    """
+
+    code = f"{BASE_CODE}-VIEW-002"
+    issue_type = ConsistencyError
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+
+        # Single loop over all views
+        for view_ref in self.validation_resources.merged_data_model.views or []:
+            if not (view := self.validation_resources.expand_view_properties(view_ref)):
+                raise RuntimeError(f"{type(self).__name__}: View {view_ref!s} not found. This is a bug in NEAT.")
+
+            if view.properties:
+                count = len(
+                    {
+                        prop.container
+                        for prop in view.properties.values()
+                        if (isinstance(prop, ViewCorePropertyRequest) and prop.container)
+                    }
+                )
+                if count > self.validation_resources.limits.views.containers:
+                    errors.append(
+                        ConsistencyError(
+                            message=(
+                                f"View {view_ref!s} references "
+                                f"{count} containers, which exceeds the limit of "
+                                f"{self.validation_resources.limits.views.containers} containers per view."
+                            ),
+                            code=self.code,
+                        )
+                    )
+
+        return errors
+
+
+class ViewImplementsCountIsOutOfLimits(DataModelValidator):
+    """Validates that a view does not implement too many other views.
+
+    ## What it does
+    Checks that the view implements no more views than the CDF limit allows.
+
+    ## Why is this bad?
+    CDF enforces limits on the number of implemented views to prevent overly deep inheritance hierarchies.
+
+    ## Example
+    If a view implements 15 other views and the CDF limit is 10 implemented views per view,
+    this validator will raise a ConsistencyError issue.
+    """
+
+    code = f"{BASE_CODE}-VIEW-003"
+    issue_type = ConsistencyError
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+
+        # Single loop over all views
+        for view_ref in self.validation_resources.merged_data_model.views or []:
+            ancestors = self.validation_resources.view_ancestors(view_ref)
+            if ancestors and len(ancestors) > self.validation_resources.limits.views.implements:
+                errors.append(
+                    ConsistencyError(
+                        message=(
+                            f"View {view_ref!s} implements {len(ancestors)} views,"
+                            " which exceeds the limit of"
+                            f" {self.validation_resources.limits.views.implements} implemented views per view."
+                        ),
+                        code=self.code,
+                    )
+                )
+        return errors
+
+
+### Container level limits
+
+
+class ContainerPropertyCountIsOutOfLimits(DataModelValidator):
+    """Validates that a container does not exceed the maximum number of properties.
+
+    ## What it does
+    Checks that the container has no more properties than the CDF limit allows.
+
+    ## Why is this bad?
+    CDF enforces limits on the number of properties per container to ensure optimal performance
+    and prevent PostGres tables that have too many columns.
+
+    ## Example
+    If a container has 150 properties and the CDF limit is 100 properties per container,
+    this validator will raise a ConsistencyError issue.
+    """
+
+    code = f"{BASE_CODE}-CONTAINER-001"
+    issue_type = ConsistencyError
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+        # Single loop over all containers
+        for container_ref in self.validation_resources.merged.containers:
+            container = self.validation_resources.select_container(container_ref)
+
+            if not container:
+                raise RuntimeError(
+                    f"{type(self).__name__}: Container {container_ref!s} "
+                    "not found in local resources. This is a bug in NEAT."
+                )
+
+            if (
+                container.properties
+                and len(container.properties) > self.validation_resources.limits.containers.properties()
+            ):
+                errors.append(
+                    ConsistencyError(
+                        message=(
+                            f"Container {container_ref!s} has {len(container.properties)} properties, "
+                            "which exceeds the limit of "
+                            f"{self.validation_resources.limits.containers.properties()} properties per container."
+                        ),
+                        fix="Define at least one property for container",
+                        code=self.code,
+                    )
+                )
+            elif not container.properties:
+                errors.append(
+                    ConsistencyError(
+                        message=(f"Container {container_ref!s} does not have any properties defined."),
+                        fix="Define at least one property for container",
+                        code=self.code,
+                    )
+                )
+
+        return errors
+
+
+class ContainerPropertyListSizeIsOutOfLimits(DataModelValidator):
+    """Validates that container property list sizes do not exceed CDF limits.
+
+    ## What it does
+    Checks that list-type properties (max_list_size) do not exceed the appropriate limit based on:
+    - Data type (Int32, Int64, DirectRelation, etc.)
+    - Presence of btree index
+    - Default vs maximum limits
+
+    ## Why is this bad?
+    CDF enforces different list size limits for different data types and indexing configurations
+    to ensure optimal performance and prevent resource exhaustion.
+
+    ## Example
+    If a DirectRelation property has max_list_size=2000 with a btree index, but the limit
+    is 1000 for indexed DirectRelations, this validator will raise a ConsistencyError issue.
+
+    ## Note
+    Enum properties are skipped as they have a separate 32-value limit checked during read time of data model to neat
+    as a SyntaxError check.
+    """
+
+    code = f"{BASE_CODE}-CONTAINER-002"
+    issue_type = ConsistencyError
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+
+        # Single loop over all containers
+        for container_ref in self.validation_resources.merged.containers:
+            container = self.validation_resources.select_container(container_ref)
+
+            if not container:
+                raise RuntimeError(
+                    f"{type(self).__name__}: Container {container_ref!s} "
+                    "not found in local resources. This is a bug in NEAT."
+                )
+
+            properties_by_index_type = self.container_property_by_index_type(container)
+
+            for property_id, property_ in container.properties.items():
+                type_ = property_.type
+
+                # Skip enum properties (have separate 32-value limit)
+                if isinstance(type_, EnumProperty):
+                    continue
+
+                # Only check listable properties with max_list_size set
+                if not isinstance(type_, ListablePropertyTypeDefinition) or type_.max_list_size is None:
+                    continue
+
+                has_btree_index = property_id in properties_by_index_type[BtreeIndex.model_fields["index_type"].default]
+                limit = self.validation_resources.limits.containers.properties.listable(type_, has_btree_index)
+
+                if type_.max_list_size > limit:
+                    errors.append(
+                        ConsistencyError(
+                            message=(
+                                f"Container {container_ref!s} has property {property_id} with list size "
+                                f"{type_.max_list_size}, which exceeds the limit of {limit} "
+                                f"for data type {type_.__class__.__name__}."
+                            ),
+                            code=self.code,
+                        )
+                    )
+
+        return errors
+
+    @staticmethod
+    def container_property_by_index_type(container: ContainerRequest) -> dict[Literal["btree", "inverted"], list]:
+        """Map container properties to their index types for limit validation.
+
+        Categorizes container properties by their index configuration:
+        - "btree": Properties with btree indexes (have stricter list size limits)
+        - "inverted": Properties with inverted indexes
+
+        This mapping is used to determine the appropriate list size limit for
+        each property based on whether it has a btree index.
+
+        Args:
+            container: The container to analyze.
+
+        Returns:
+            Dictionary with index type strings as keys and lists of property identifiers
+            as values. Returns empty lists for both index types if container has no indexes.
+        """
+
+        container_property_by_index_type: dict[Literal["btree", "inverted"], list] = {
+            BtreeIndex.model_fields["index_type"].default: [],
+            InvertedIndex.model_fields["index_type"].default: [],
+        }
+        if not container.indexes:
+            return container_property_by_index_type
+
+        for index in container.indexes.values():
+            if isinstance(index, BtreeIndex):
+                container_property_by_index_type[BtreeIndex.model_fields["index_type"].default].extend(index.properties)
+            elif isinstance(index, InvertedIndex):
+                container_property_by_index_type[InvertedIndex.model_fields["index_type"].default].extend(
+                    index.properties
+                )
+
+        return container_property_by_index_type