cognite-neat 0.123.2__py3-none-any.whl → 0.127.30__py3-none-any.whl

cognite/neat/_data_model/validation/dms/_ai_readiness.py

@@ -0,0 +1,364 @@
+"""Validators for checking if data model is AI-ready."""
+
+from cognite.neat._data_model.models.dms._data_types import EnumProperty
+from cognite.neat._data_model.validation.dms._base import DataModelValidator
+from cognite.neat._issues import Recommendation
+
+BASE_CODE = "NEAT-DMS-AI-READINESS"
+
+
+class DataModelMissingName(DataModelValidator):
+    """Validates that data model has a human-readable name.
+
+    ## What it does
+    Validates that the data model has a human-readable name.
+
+    ## Why is this bad?
+    Often the data model ids are technical identifiers, abbreviations, etc.
+    A missing name makes it harder for users (humans or machines) to understand what the data model represents.
+    Providing a clear name improves usability, maintainability, searchability, and AI-readiness.
+
+    ## Example
+    A data model has an id IEC61400-25-2 but no name. Users may find it difficult to understand what this data model
+    represents. However adding a name "Wind Energy Information Model" would increase clarity and usability.
+    """
+
+    code = f"{BASE_CODE}-001"
+    issue_type = Recommendation
+
+    def run(self) -> list[Recommendation]:
+        recommendations: list[Recommendation] = []
+
+        if not self.local_resources.data_model_name:
+            recommendations.append(
+                Recommendation(
+                    message="Data model is missing a human-readable name.",
+                    fix="Add a clear and concise name to the data model.",
+                    code=self.code,
+                )
+            )
+
+        return recommendations
+
+
+class DataModelMissingDescription(DataModelValidator):
+    """Validates that data model has a human-readable description.
+
+    ## What it does
+    Validates that the data model has a human-readable description.
+
+    ## Why is this bad?
+    A missing description makes it harder for users (humans or machines) to understand the purpose and scope
+    of the data model. The description provides important context about what domain the data model covers,
+    what use cases it supports, and how it should be used.
+
+    ## Example
+    A data model has an id CIM, with name Common Information Model, but no description. Users may find it difficult to
+    understand what this data model represents, unless extra context is provided. In this particualar case, name
+    does not provide sufficient information, as it is too generic, that this data model is focused on the
+    electrical power systems domain. However, providing a description such as:
+    "The Common Information Model (CIM) is a standard developed by IEC for representing power system
+    components and their relationships. It is widely used in the electrical utility industry for data
+    exchange and system modeling." would greatly improve clarity and usability.
+    """
+
+    code = f"{BASE_CODE}-002"
+    issue_type = Recommendation
+
+    def run(self) -> list[Recommendation]:
+        recommendations: list[Recommendation] = []
+
+        if not self.local_resources.data_model_description:
+            recommendations.append(
+                Recommendation(
+                    message="Data model is missing a description.",
+                    fix="Add a clear and concise description to the data model.",
+                    code=self.code,
+                )
+            )
+
+        return recommendations
+
+
+class ViewMissingName(DataModelValidator):
+    """Validates that a View has a human-readable name.
+
+    ## What it does
+    Validates that each view in the data model has a human-readable name.
+
+    ## Why is this bad?
+    A missing name makes it harder for users (humans or machines) to understand the purpose of the view.
+    This is important as views' external ids are often based on technical identifiers, abbreviations, etc.
+    Providing a clear name improves usability, maintainability, searchability, and AI-readiness.
+
+    ## Example
+    A view has an id CFIHOS-30000038 but no name. Users may find it difficult to understand what this view represents,
+    unless they look up the id in documentation or other resources. Adding name "Pump" would increase clarity and
+    usability.
+    """
+
+    code = f"{BASE_CODE}-003"
+    issue_type = Recommendation
+
+    def run(self) -> list[Recommendation]:
+        recommendations: list[Recommendation] = []
+
+        for view_ref in self.local_resources.data_model_views:
+            view = self.local_resources.views_by_reference.get(view_ref)
+
+            if view is None:
+                raise RuntimeError(f"View {view_ref!s} not found in local resources. This is a bug.")
+
+            if not view.name:
+                recommendations.append(
+                    Recommendation(
+                        message=f"View {view_ref!s} is missing a human-readable name.",
+                        fix="Add a clear and concise name to the view.",
+                        code=self.code,
+                    )
+                )
+
+        return recommendations
+
+
+class ViewMissingDescription(DataModelValidator):
+    """Validates that a View has a human-readable description.
+
+    ## What it does
+    Validates that each view in the data model has a human-readable description.
+
+    ## Why is this bad?
+    A missing description makes it harder for users (humans or machines) to understand in what context the view
+    should be used. The description can provide important information about the view's purpose, scope, and usage.
+
+
+    ## Example
+    A view Site has no description. Users may find it difficult to understand what this view represents, unless
+    extra context is provided. Even if we know that Site is used in the context of wind energy developments, a
+    description is necessary as it can be used in various context within the same domain such as:
+
+    Option 1 — Project area
+    This view represents a geographical area where wind energy projects are developed and managed.
+
+    Option 2 — Lease area
+    The legally defined lease area allocated for offshore wind development.
+
+    Option 3 — Measurement site
+    A specific location where wind measurements (e.g., LiDAR, met mast) are collected.
+
+    """
+
+    code = f"{BASE_CODE}-004"
+    issue_type = Recommendation
+
+    def run(self) -> list[Recommendation]:
+        recommendations: list[Recommendation] = []
+
+        for view_ref in self.local_resources.data_model_views:
+            view = self.local_resources.views_by_reference.get(view_ref)
+
+            if view is None:
+                raise RuntimeError(f"View {view_ref!s} not found in local resources. This is a bug.")
+
+            if not view.description:
+                recommendations.append(
+                    Recommendation(
+                        message=f"View {view_ref!s} is missing a description.",
+                        fix="Add a clear and concise description to the view.",
+                        code=self.code,
+                    )
+                )
+
+        return recommendations
+
+
+class ViewPropertyMissingName(DataModelValidator):
+    """Validates that a view property has a human-readable name.
+
+    ## What it does
+    Validates that each view property in the data model has a human-readable name.
+
+    ## Why is this bad?
+    A missing name makes it harder for users (humans or machines) to understand the purpose of the view property.
+    This is important as view property's ids are often based on technical identifiers, abbreviations, etc.
+    Providing a clear name improves usability, maintainability, searchability, and AI-readiness.
+
+    ## Example
+    A view WindTurbine has a property pc which has no name. Users may find it difficult to understand what this view
+    property represents, unless they look up the id in documentation or other resources. Adding name "power curve"
+    would increase clarity and usability.
+    """
+
+    code = f"{BASE_CODE}-005"
+    issue_type = Recommendation
+
+    def run(self) -> list[Recommendation]:
+        recommendations: list[Recommendation] = []
+
+        for view_ref in self.local_resources.data_model_views:
+            if properties := self.local_resources.properties_by_view.get(view_ref):
+                for prop_ref, definition in properties.items():
+                    if not definition.name:
+                        recommendations.append(
+                            Recommendation(
+                                message=f"View {view_ref!s} property {prop_ref!s} is missing a human-readable name.",
+                                fix="Add a clear and concise name to the view property.",
+                                code=self.code,
+                            )
+                        )
+
+        return recommendations
+
+
+class ViewPropertyMissingDescription(DataModelValidator):
+    """Validates that a View property has a human-readable description.
+
+    ## What it does
+    Validates that each view property in the data model has a human-readable description.
+
+    ## Why is this bad?
+    A missing description makes it harder for users (humans or machines) to understand in what context the view property
+    should be used. The description can provide important information about the view property's purpose,
+    scope, and usage.
+
+
+    ## Example
+    A view WindTurbine has a property status with no description. Users may find it difficult to understand what this
+    property represents, unless extra context is provided. Even if we know that status is related to wind turbine
+    operations, a description is necessary as it can have different meanings in various contexts:
+
+    Option 1 — Operational status
+    Current operational state of the wind turbine (e.g., running, stopped, maintenance, fault).
+
+    Option 2 — Connection status
+    Grid connection status indicating whether the turbine is connected to the electrical grid.
+
+    Option 3 — Availability status
+    Availability state for production indicating whether the turbine is available for power generation.
+
+    """
+
+    code = f"{BASE_CODE}-006"
+    issue_type = Recommendation
+
+    def run(self) -> list[Recommendation]:
+        recommendations: list[Recommendation] = []
+
+        for view_ref in self.local_resources.data_model_views:
+            if properties := self.local_resources.properties_by_view.get(view_ref):
+                for prop_ref, definition in properties.items():
+                    if not definition.description:
+                        recommendations.append(
+                            Recommendation(
+                                message=f"View {view_ref!s} property {prop_ref!s} is missing a description.",
+                                fix="Add a clear and concise description to the view property.",
+                                code=self.code,
+                            )
+                        )
+
+        return recommendations
+
+
+class EnumerationMissingName(DataModelValidator):
+    """Validates that an enumeration has a human-readable name.
+
+    ## What it does
+    Validates that each enumeration value in the data model has a human-readable name.
+
+    ## Why is this bad?
+    A missing name makes it harder for users (humans or machines) to understand the purpose of the enumeration value.
+    This is important as enumeration values are often technical codes or abbreviations, and a clear name improves
+    usability, maintainability, searchability, and AI-readiness.
+
+    ## Example
+    An enumeration value with id "NOM" in a wind turbine operational mode property has no name. Users may find it
+    difficult to understand what this value represents. Adding name "Normal Operation" would increase clarity
+    and usability.
+    """
+
+    code = f"{BASE_CODE}-007"
+    issue_type = Recommendation
+
+    def run(self) -> list[Recommendation]:
+        recommendations: list[Recommendation] = []
+
+        for container_ref, container in self.local_resources.containers_by_reference.items():
+            if container_ref.space != self.local_resources.data_model_reference.space:
+                continue
+
+            for prop_ref, definition in container.properties.items():
+                if not isinstance(definition.type, EnumProperty):
+                    continue
+
+                for value, enum_def in definition.type.values.items():
+                    if not enum_def.name:
+                        recommendations.append(
+                            Recommendation(
+                                message=(
+                                    f"Enumeration value {value!r} in property {prop_ref!s} of container "
+                                    f"{container_ref!s} is missing a human-readable name."
+                                ),
+                                fix="Add a clear and concise name to the enumeration value.",
+                                code=self.code,
+                            )
+                        )
+
+        return recommendations
+
+
+class EnumerationMissingDescription(DataModelValidator):
+    """Validates that an enumeration value has a human-readable description.
+
+    ## What it does
+    Validates that each enumeration value in the data model has a human-readable description.
+
+    ## Why is this bad?
+    A missing description makes it harder for users (humans or machines) to understand the meaning and context
+    of the enumeration value. The description can provide important information about when and how the value
+    should be used, especially when enumeration values are technical codes or abbreviations.
+
+    ## Example
+    An enumeration value "NOM" in a wind turbine operational mode property has no description. Users may find it
+    difficult to understand what this value represents without additional context. Even with a name like
+    "Normal Operation", the description is valuable as it can clarify specifics:
+
+    Option 1 — Basic definition
+    The turbine is operating normally and generating power according to its power curve.
+
+    Option 2 — Detailed operational context
+    The turbine is in normal operation mode, actively generating power with all systems functioning within
+    specified parameters and connected to the grid.
+
+    Option 3 — Contrasting with other modes
+    Standard operating mode where the turbine follows the power curve and responds to grid commands,
+    as opposed to maintenance mode or fault conditions.
+    """
+
+    code = f"{BASE_CODE}-008"
+    issue_type = Recommendation
+
+    def run(self) -> list[Recommendation]:
+        recommendations: list[Recommendation] = []
+
+        for container_ref, container in self.local_resources.containers_by_reference.items():
+            if container_ref.space != self.local_resources.data_model_reference.space:
+                continue
+
+            for prop_ref, definition in container.properties.items():
+                if not isinstance(definition.type, EnumProperty):
+                    continue
+
+                for value, enum_def in definition.type.values.items():
+                    if not enum_def.description:
+                        recommendations.append(
+                            Recommendation(
+                                message=(
+                                    f"Enumeration value {value!r} in property {prop_ref!s} of container "
+                                    f"{container_ref!s} is missing a human-readable description."
+                                ),
+                                fix="Add a clear and concise description to the enumeration value.",
+                                code=self.code,
+                            )
+                        )
+
+        return recommendations

cognite/neat/_data_model/validation/dms/_base.py

@@ -0,0 +1,307 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from itertools import chain
+from typing import ClassVar, TypeAlias, cast
+
+from pyparsing import cached_property
+
+from cognite.neat._data_model.models.dms._container import ContainerRequest
+from cognite.neat._data_model.models.dms._references import (
+    ContainerDirectReference,
+    ContainerReference,
+    DataModelReference,
+    ViewDirectReference,
+    ViewReference,
+)
+from cognite.neat._data_model.models.dms._view_property import ViewRequestProperty
+from cognite.neat._data_model.models.dms._views import ViewRequest
+from cognite.neat._issues import ConsistencyError, Recommendation
+from cognite.neat._utils.useful_types import ModusOperandi
+
+# Type aliases for better readability
+ViewsByReference: TypeAlias = dict[ViewReference, ViewRequest]
+ContainersByReference: TypeAlias = dict[ContainerReference, ContainerRequest]
+AncestorsByReference: TypeAlias = dict[ViewReference, set[ViewReference]]
+ReverseToDirectMapping: TypeAlias = dict[
+    tuple[ViewReference, str], tuple[ViewReference, ContainerDirectReference | ViewDirectReference]
+]
+ConnectionEndNodeTypes: TypeAlias = dict[tuple[ViewReference, str], ViewReference | None]
+
+
+@dataclass
+class LocalResources:
+    """Local data model resources."""
+
+    data_model_reference: DataModelReference
+    data_model_views: set[ViewReference]
+    data_model_description: str | None
+    data_model_name: str | None
+    views_by_reference: ViewsByReference
+    properties_by_view: dict[ViewReference, dict[str, ViewRequestProperty]]
+    ancestors_by_view_reference: AncestorsByReference
+    reverse_to_direct_mapping: ReverseToDirectMapping
+    containers_by_reference: ContainersByReference
+    connection_end_node_types: ConnectionEndNodeTypes
+
+
+@dataclass
+class CDFResources:
+    """CDF resources."""
+
+    views_by_reference: ViewsByReference
+    ancestors_by_view_reference: AncestorsByReference
+    containers_by_reference: ContainersByReference
+    data_model_views: set[ViewReference]
+
+
+class DataModelValidator(ABC):
+    """Assessors for fundamental data model principles."""
+
+    code: ClassVar[str]
+    issue_type: ClassVar[type[ConsistencyError] | type[Recommendation]]
+
+    def __init__(
+        self,
+        local_resources: LocalResources,
+        cdf_resources: CDFResources,
+        modus_operandi: ModusOperandi = "additive",
+    ) -> None:
+        self.local_resources = local_resources
+        self.cdf_resources = cdf_resources
+        self.modus_operandi = modus_operandi
+
+    @abstractmethod
+    def run(self) -> list[ConsistencyError] | list[Recommendation] | list[ConsistencyError | Recommendation]:
+        """Execute the success handler on the data model."""
+        # do something with data model
+        ...
+
+    def _select_view_with_property(self, view_ref: ViewReference, property_: str) -> ViewRequest | None:
+        """Select the appropriate view (local or CDF) that contains desired property.
+
+        Prioritizes views that contain the property  (first local than CDF),
+        then falls back to any available view (even without the property).
+
+        Args:
+            view_ref: Reference to the view.
+            property_: Property name to look for.
+
+        Returns:
+            The selected ViewRequest if found, else None.
+
+        !! note "Behavior based on modus operandi"
+            - In "additive" modus operandi, local and CDF view will be considered irrirespective of their space.
+            - In "rebuild" modus operandi, local views will be considered irrispective of their space, while CDF views
+              will only be considered if they belong to the different space than the local data model space
+              (as they are considered external resources that is managed under other data model/schema space).
+
+        """
+
+        local_view = self.local_resources.views_by_reference.get(view_ref)
+        cdf_view = (
+            self.cdf_resources.views_by_reference.get(view_ref)
+            if view_ref.space != self.local_resources.data_model_reference.space or self.modus_operandi == "additive"
+            else None
+        )
+
+        # Try views with the property first, then any available view
+        candidates = chain(
+            (v for v in (local_view, cdf_view) if v and v.properties and property_ in v.properties),
+            (v for v in (local_view, cdf_view) if v),
+        )
+
+        return next(candidates, None)
+
+    def _select_container_with_property(
+        self, container_ref: ContainerReference, property_: str
+    ) -> ContainerRequest | None:
+        """Select the appropriate container (local or CDF) that contains the desired property.
+
+        Prioritizes containers that contain the property (first local than CDF),
+        then falls back to any available container.
+
+        Args:
+            container_ref: Reference to the container.
+            property_: Property name to look for.
+
+        Returns:
+            The selected ContainerRequest if found, else None.
+
+        !! note "Behavior based on modus operandi"
+            - In "additive" modus operandi, local and CDF containers will be considered irrirespective of their space.
+            - In "rebuild" modus operandi, local containers will be considered irrispective of their space, while CDF
+              containers will only be considered if they belong to the different space than the local data model space
+              (as they are considered external resources that is managed under other data model/schema space).
+
+        """
+        local_container = self.local_resources.containers_by_reference.get(container_ref)
+        cdf_container = self.cdf_resources.containers_by_reference.get(container_ref)
+
+        cdf_container = (
+            self.cdf_resources.containers_by_reference.get(container_ref)
+            if container_ref.space != self.local_resources.data_model_reference.space
+            or self.modus_operandi == "additive"
+            else None
+        )
+
+        # Try containers with the property first, then any available container
+        candidates = chain(
+            (c for c in (local_container, cdf_container) if c and c.properties and property_ in c.properties),
+            (c for c in (local_container, cdf_container) if c),
+        )
+
+        return next(candidates, None)
+
+    @cached_property
+    def data_model_view_references(self) -> set[ViewReference]:
+        """Get all data model view references to validate based on deployment mode.
+
+        In "rebuild" mode, returns only local data model view references.
+        In "additive" mode, returns union of local and CDF data model view references.
+
+        Returns:
+            Set of ViewReference objects representing all data model views to validate.
+        """
+        return (
+            self.local_resources.data_model_views.union(self.cdf_resources.data_model_views)
+            if self.modus_operandi == "additive"
+            else self.local_resources.data_model_views
+        )
+
+    @cached_property
+    def views_references(self) -> set[ViewReference]:
+        """Get all view references to validate based on deployment mode.
+
+        In "rebuild" mode, returns only local view references.
+        In "additive" mode, returns union of local and CDF view references.
+
+        Returns:
+            Set of ViewReference objects representing all views to validate.
+        """
+
+        return (
+            set(self.local_resources.views_by_reference.keys()).union(set(self.cdf_resources.views_by_reference.keys()))
+            if self.modus_operandi == "additive"
+            else set(self.local_resources.views_by_reference.keys())
+        )
+
+    @cached_property
+    def container_references(self) -> set[ContainerReference]:
+        """Get all container references to validate based on deployment mode.
+
+        In "rebuild" mode, returns only local container references.
+        In "additive" mode, returns union of local and CDF container references.
+
+        Returns:
+            Set of ContainerReference objects representing all containers to validate.
+        """
+
+        return (
+            set(self.local_resources.containers_by_reference.keys()).union(
+                set(self.cdf_resources.containers_by_reference.keys())
+            )
+            if self.modus_operandi == "additive"
+            else set(self.local_resources.containers_by_reference.keys())
+        )
+
+    @cached_property
+    def merged_views(self) -> dict[ViewReference, ViewRequest]:
+        """Get views with merged properties and implements for accurate limit checking.
+
+        In "rebuild" mode, returns only local views.
+        In "additive" mode, merges local and CDF views by:
+        - Combining properties from both versions (local overrides CDF)
+        - Combining implements lists (union, no duplicates)
+        - Using CDF view as base if it exists, otherwise local view
+
+        This ensures limit validation accounts for the actual deployed state
+        in additive deployments.
+
+        Returns:
+            Dictionary mapping ViewReference to merged ViewRequest objects.
+
+        Raises:
+            RuntimeError: If a referenced view is not found in either local or CDF resources.
+        """
+
+        if self.modus_operandi != "additive":
+            return self.local_resources.views_by_reference
+
+        merged_views: dict[ViewReference, ViewRequest] = {}
+        # Merge local views, combining properties if view exists in both
+        for view_ref in self.views_references:
+            cdf_view = self.cdf_resources.views_by_reference.get(view_ref)
+            local_view = self.local_resources.views_by_reference.get(view_ref)
+
+            if not cdf_view and not local_view:
+                raise RuntimeError(f"View {view_ref!s} not found in either local or CDF resources. This is a bug!")
+
+            # this will later update of local properties and implements
+            merged_views[view_ref] = cast(ViewRequest, (cdf_view or local_view)).model_copy(deep=True)
+
+            if local_view and local_view.properties:
+                if not merged_views[view_ref].properties:
+                    merged_views[view_ref].properties = local_view.properties
+                else:
+                    merged_views[view_ref].properties.update(local_view.properties)
+
+            if local_view and local_view.implements:
+                if not merged_views[view_ref].implements:
+                    merged_views[view_ref].implements = local_view.implements
+                else:  # mypy is complaining here about possible None which is not possible due to the check above
+                    for impl in local_view.implements:
+                        if impl not in cast(list[ViewReference], merged_views[view_ref].implements):
+                            cast(list[ViewReference], merged_views[view_ref].implements).append(impl)
+
+        return merged_views
+
+    @cached_property
+    def merged_containers(self) -> dict[ContainerReference, ContainerRequest]:
+        """Get containers with merged properties for accurate limit checking.
+
+        In "rebuild" mode, returns only local containers.
+        In "additive" mode, merges local and CDF containers by:
+        - Combining properties from both versions (local overrides CDF)
+        - Using CDF container as base if it exists, otherwise local container
+
+        This ensures limit validation accounts for the actual deployed state
+        in additive deployments.
+
+        Returns:
+            Dictionary mapping ContainerReference to merged ContainerRequest objects.
+
+        Raises:
+            RuntimeError: If a referenced container is not found in either local or CDF resources.
+        """
+
+        if self.modus_operandi != "additive":
+            return self.local_resources.containers_by_reference
+
+        merged_containers: dict[ContainerReference, ContainerRequest] = {}
+        # Merge local containers, combining properties if container exists in both
+        for container_ref in self.container_references:
+            cdf_container = self.cdf_resources.containers_by_reference.get(container_ref)
+            local_container = self.local_resources.containers_by_reference.get(container_ref)
+
+            if not cdf_container and not local_container:
+                raise RuntimeError(
+                    f"Container {container_ref!s} not found in either local or CDF resources. This is a bug!"
+                )
+
+            merged_containers[container_ref] = cast(ContainerRequest, (cdf_container or local_container)).model_copy(
+                deep=True
+            )
+
+            if local_container and local_container.properties:
+                if not merged_containers[container_ref].properties:
+                    merged_containers[container_ref].properties = local_container.properties
+                else:
+                    merged_containers[container_ref].properties.update(local_container.properties)
+
+        return merged_containers
+
+    def _is_ancestor_of_target(self, potential_ancestor: ViewReference, target_view_ref: ViewReference) -> bool:
+        """Check if a view is an ancestor of the target view."""
+        return potential_ancestor in self.local_resources.ancestors_by_view_reference.get(
+            target_view_ref, set()
+        ) or potential_ancestor in self.cdf_resources.ancestors_by_view_reference.get(target_view_ref, set())