cognite-neat 0.123.2__py3-none-any.whl → 0.127.30__py3-none-any.whl

cognite/neat/_data_model/validation/dms/_orchestrator.py

@@ -0,0 +1,222 @@
+from collections.abc import Callable
+from itertools import chain
+
+from cognite.neat._client import NeatClient
+from cognite.neat._data_model._analysis import DataModelAnalysis
+from cognite.neat._data_model._shared import OnSuccessIssuesChecker
+from cognite.neat._data_model.models.dms._container import ContainerRequest
+from cognite.neat._data_model.models.dms._limits import SchemaLimits
+from cognite.neat._data_model.models.dms._references import ContainerReference, DataModelReference, ViewReference
+from cognite.neat._data_model.models.dms._schema import RequestSchema
+from cognite.neat._data_model.models.dms._view_property import ViewCorePropertyRequest
+from cognite.neat._data_model.models.dms._views import ViewRequest
+from cognite.neat._data_model.validation.dms._limits import (
+    ContainerPropertyCountIsOutOfLimits,
+    ContainerPropertyListSizeIsOutOfLimits,
+    DataModelViewCountIsOutOfLimits,
+    ViewContainerCountIsOutOfLimits,
+    ViewImplementsCountIsOutOfLimits,
+    ViewPropertyCountIsOutOfLimits,
+)
+from cognite.neat._utils.useful_types import ModusOperandi
+
+from ._ai_readiness import (
+    DataModelMissingDescription,
+    DataModelMissingName,
+    EnumerationMissingDescription,
+    EnumerationMissingName,
+    ViewMissingDescription,
+    ViewMissingName,
+    ViewPropertyMissingDescription,
+    ViewPropertyMissingName,
+)
+from ._base import CDFResources, DataModelValidator, LocalResources
+from ._connections import (
+    ConnectionValueTypeUndefined,
+    ConnectionValueTypeUnexisting,
+    ReverseConnectionContainerMissing,
+    ReverseConnectionContainerPropertyMissing,
+    ReverseConnectionContainerPropertyWrongType,
+    ReverseConnectionPointsToAncestor,
+    ReverseConnectionSourcePropertyMissing,
+    ReverseConnectionSourcePropertyWrongType,
+    ReverseConnectionSourceViewMissing,
+    ReverseConnectionTargetMismatch,
+    ReverseConnectionTargetMissing,
+)
+from ._consistency import ViewSpaceVersionInconsistentWithDataModel
+from ._containers import (
+    ExternalContainerDoesNotExist,
+    ExternalContainerPropertyDoesNotExist,
+    RequiredContainerDoesNotExist,
+)
+from ._views import ImplementedViewNotExisting, ViewToContainerMappingNotPossible
+
+
+class DmsDataModelValidation(OnSuccessIssuesChecker):
+    """Placeholder for DMS Quality Assessment functionality."""
+
+    def __init__(
+        self,
+        client: NeatClient | None = None,
+        modus_operandi: ModusOperandi = "additive",
+        can_run_validator: Callable[[str, type], bool] | None = None,
+    ) -> None:
+        super().__init__()
+        self._client = client
+        self._can_run_validator = can_run_validator or (lambda code, issue_type: True)  # type: ignore
+        self._modus_operandi = modus_operandi
+        self._has_run = False
+
+    def _gather_resources(self, data_model: RequestSchema) -> tuple[LocalResources, CDFResources, SchemaLimits]:
+        """Gather local and CDF resources needed for validation."""
+
+        analysis = DataModelAnalysis(data_model)
+
+        local_views_by_reference = analysis.view_by_reference(include_inherited_properties=True)
+        local_ancestors_by_view_reference = analysis.ancestors_by_view(list(local_views_by_reference.values()))
+        local_reverse_to_direct_mapping = analysis.reverse_to_direct_mapping
+        local_containers_by_reference = analysis.container_by_reference
+        local_data_model_views = set(data_model.data_model.views) if data_model.data_model.views else set()
+
+        local_resources = LocalResources(
+            data_model_description=data_model.data_model.description,
+            data_model_name=data_model.data_model.name,
+            data_model_reference=data_model.data_model.as_reference(),
+            views_by_reference=local_views_by_reference,
+            properties_by_view=analysis.properties_by_view,
+            ancestors_by_view_reference=local_ancestors_by_view_reference,
+            reverse_to_direct_mapping=local_reverse_to_direct_mapping,
+            containers_by_reference=local_containers_by_reference,
+            connection_end_node_types=analysis.connection_end_node_types,
+            data_model_views=local_data_model_views,
+        )
+
+        cdf_views_by_reference = self._cdf_view_by_reference(
+            list(analysis.referenced_views(include_connection_end_node_types=True)),
+            include_inherited_properties=True,
+        )
+        cdf_ancestors_by_view_reference = analysis.ancestors_by_view(list(cdf_views_by_reference.values()))
+        cdf_containers_by_reference = self._cdf_container_by_reference(
+            list(self._referenced_containers(local_views_by_reference, cdf_views_by_reference))
+        )
+        cdf_data_model_views = self._cdf_data_model_views(data_model.data_model.as_reference())
+
+        cdf_resources = CDFResources(
+            views_by_reference=cdf_views_by_reference,
+            ancestors_by_view_reference=cdf_ancestors_by_view_reference,
+            containers_by_reference=cdf_containers_by_reference,
+            data_model_views=cdf_data_model_views,
+        )
+
+        return local_resources, cdf_resources, self._cdf_limits()
+
+    def run(self, data_model: RequestSchema) -> None:
+        """Run quality assessment on the DMS data model."""
+
+        # Helper wrangled data model components
+        local_resources, cdf_resources, cdf_limits = self._gather_resources(data_model)
+
+        # Initialize all validators
+        validators: list[DataModelValidator] = [
+            # Limits
+            DataModelViewCountIsOutOfLimits(local_resources, cdf_resources, cdf_limits, self._modus_operandi),
+            ViewPropertyCountIsOutOfLimits(local_resources, cdf_resources, cdf_limits, self._modus_operandi),
+            ViewImplementsCountIsOutOfLimits(local_resources, cdf_resources, cdf_limits, self._modus_operandi),
+            ViewContainerCountIsOutOfLimits(local_resources, cdf_resources, cdf_limits, self._modus_operandi),
+            ContainerPropertyCountIsOutOfLimits(local_resources, cdf_resources, cdf_limits, self._modus_operandi),
+            ContainerPropertyListSizeIsOutOfLimits(local_resources, cdf_resources, cdf_limits, self._modus_operandi),
+            # Views
+            ViewToContainerMappingNotPossible(local_resources, cdf_resources, self._modus_operandi),
+            ImplementedViewNotExisting(local_resources, cdf_resources, self._modus_operandi),
+            # Containers
+            ExternalContainerDoesNotExist(local_resources, cdf_resources, self._modus_operandi),
+            ExternalContainerPropertyDoesNotExist(local_resources, cdf_resources, self._modus_operandi),
+            RequiredContainerDoesNotExist(local_resources, cdf_resources, self._modus_operandi),
+            # Consistency
+            ViewSpaceVersionInconsistentWithDataModel(local_resources, cdf_resources, self._modus_operandi),
+            # Connections
+            ConnectionValueTypeUnexisting(local_resources, cdf_resources, self._modus_operandi),
+            ConnectionValueTypeUndefined(local_resources, cdf_resources, self._modus_operandi),
+            ReverseConnectionContainerMissing(local_resources, cdf_resources, self._modus_operandi),
+            ReverseConnectionContainerPropertyMissing(local_resources, cdf_resources, self._modus_operandi),
+            ReverseConnectionContainerPropertyWrongType(local_resources, cdf_resources, self._modus_operandi),
+            ReverseConnectionSourceViewMissing(local_resources, cdf_resources, self._modus_operandi),
+            ReverseConnectionSourcePropertyMissing(local_resources, cdf_resources, self._modus_operandi),
+            ReverseConnectionSourcePropertyWrongType(local_resources, cdf_resources, self._modus_operandi),
+            ReverseConnectionPointsToAncestor(local_resources, cdf_resources, self._modus_operandi),
+            ReverseConnectionTargetMismatch(local_resources, cdf_resources, self._modus_operandi),
+            ReverseConnectionTargetMissing(local_resources, cdf_resources, self._modus_operandi),
+            # AI Readiness
+            DataModelMissingName(local_resources, cdf_resources, self._modus_operandi),
+            DataModelMissingDescription(local_resources, cdf_resources, self._modus_operandi),
+            ViewMissingName(local_resources, cdf_resources, self._modus_operandi),
+            ViewMissingDescription(local_resources, cdf_resources, self._modus_operandi),
+            ViewPropertyMissingName(local_resources, cdf_resources, self._modus_operandi),
+            ViewPropertyMissingDescription(local_resources, cdf_resources, self._modus_operandi),
+            EnumerationMissingName(local_resources, cdf_resources, self._modus_operandi),
+            EnumerationMissingDescription(local_resources, cdf_resources, self._modus_operandi),
+        ]
+
+        # Run validators
+        for validator in validators:
+            if self._can_run_validator(validator.code, validator.issue_type):
+                self._issues.extend(validator.run())
+
+        self._has_run = True
+
+    def _cdf_data_model_views(self, data_model_ref: DataModelReference) -> set[ViewReference]:
+        """Get all data model views in CDF."""
+
+        if not self._client:
+            return set()
+
+        data_model = self._client.data_models.retrieve([data_model_ref])
+
+        return set(data_model[0].views) if data_model and data_model[0].views else set()
+
+    def _cdf_view_by_reference(
+        self, views: list[ViewReference], include_inherited_properties: bool = True
+    ) -> dict[ViewReference, ViewRequest]:
+        """Fetch view definition from CDF."""
+
+        if not self._client:
+            return {}
+        return {
+            response.as_reference(): response.as_request()
+            for response in self._client.views.retrieve(
+                views, include_inherited_properties=include_inherited_properties
+            )
+        }
+
+    def _cdf_container_by_reference(
+        self, containers: list[ContainerReference]
+    ) -> dict[ContainerReference, ContainerRequest]:
+        """Fetch container definition from CDF."""
+
+        if not self._client:
+            return {}
+        return {
+            response.as_reference(): response.as_request() for response in self._client.containers.retrieve(containers)
+        }
+
+    def _cdf_limits(self) -> SchemaLimits:
+        """Fetch DMS statistics from CDF."""
+
+        if not self._client:
+            return SchemaLimits()
+        return SchemaLimits.from_api_response(self._client.statistics.project())
+
+    def _referenced_containers(
+        self,
+        local_views_by_reference: dict[ViewReference, ViewRequest],
+        cdf_views_by_reference: dict[ViewReference, ViewRequest],
+    ) -> set[ContainerReference]:
+        """Get all referenced containers in the physical data model both local and in CDF."""
+        return {
+            property_.container
+            for view in chain(local_views_by_reference.values(), cdf_views_by_reference.values())
+            if view.properties
+            for property_ in view.properties.values()
+            if isinstance(property_, ViewCorePropertyRequest)
+        }

cognite/neat/_data_model/validation/dms/_views.py

@@ -0,0 +1,103 @@
+"""Validators for checking containers in the data model."""
+
+from cognite.neat._data_model.models.dms._view_property import ViewCorePropertyRequest
+from cognite.neat._data_model.validation.dms._base import DataModelValidator
+from cognite.neat._issues import ConsistencyError
+
+BASE_CODE = "NEAT-DMS-VIEW"
+
+
+class ViewToContainerMappingNotPossible(DataModelValidator):
+    """Validates that container and container property referenced by view property exist.
+
+    ## What it does
+    Validates that for each view property that maps to a container and container property,
+    the referenced container and container property exist.
+
+    ## Why is this bad?
+    If a view property references a container or container property that does not exist,
+    the data model cannot be deployed to CDF. This means that view property will not be able to function.
+
+    ## Example
+    View WindTurbine has property location that maps to container WindTurbineContainer and property gpsCoordinates.
+    If WindTurbineContainer and/or property gpsCoordinates does not exist, the data model cannot be deployed to CDF.
+    """
+
+    code = f"{BASE_CODE}-001"
+    issue_type = ConsistencyError
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+
+        for view_ref, view in self.local_resources.views_by_reference.items():
+            for property_ref, property_ in view.properties.items():
+                if not isinstance(property_, ViewCorePropertyRequest):
+                    continue
+
+                container_ref = property_.container
+                container_property = property_.container_property_identifier
+
+                container = self._select_container_with_property(container_ref, container_property)
+
+                if not container:
+                    errors.append(
+                        ConsistencyError(
+                            message=(
+                                f"View {view_ref!s} property {property_ref!s} maps to "
+                                f"container {container_ref!s} which does not exist."
+                            ),
+                            fix="Define necessary container",
+                            code=self.code,
+                        )
+                    )
+                elif container_property not in container.properties:
+                    errors.append(
+                        ConsistencyError(
+                            message=(
+                                f"View {view_ref!s} property {property_ref!s} maps to "
+                                f"container {container_ref!s} which does not have "
+                                f"property '{container_property}'."
+                            ),
+                            fix="Define necessary container property",
+                            code=self.code,
+                        )
+                    )
+
+        return errors
+
+
+class ImplementedViewNotExisting(DataModelValidator):
+    """Validates that implemented (inherited) view exists.
+
+    ## What it does
+    Validates that all views which are implemented (inherited) in the data model actually exist either locally
+    or in CDF.
+
+    ## Why is this bad?
+    If a view being implemented (inherited) does not exist, the data model cannot be deployed to CDF.
+
+    ## Example
+    If view WindTurbine implements (inherits) view Asset, but Asset view does not exist in the data model
+    or in CDF, the data model cannot be deployed to CDF.
+    """
+
+    code = f"{BASE_CODE}-002"
+    issue_type = ConsistencyError
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+
+        for view_ref, view in self.local_resources.views_by_reference.items():
+            if view.implements is None:
+                continue
+            for implement in view.implements:
+                if implement not in self.merged_views:
+                    errors.append(
+                        ConsistencyError(
+                            message=f"View {view_ref!s} implements {implement!s} which is not defined.",
+                            fix="Define the missing view or remove it from the implemented views list",
+                            code=self.code,
+                        )
+                    )
+
+        return errors

cognite/neat/_exceptions.py

@@ -0,0 +1,56 @@
+from abc import ABC
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from cognite.neat._issues import ModelSyntaxError
+    from cognite.neat._utils.http_client import HTTPMessage
+
+
+class NeatException(Exception, ABC):
+    """Base class for all exceptions raised by Neat."""
+
+    pass
+
+
+class DataModelImportException(NeatException):
+    """Raised when there is an error importing a model."""
+
+    def __init__(self, errors: "list[ModelSyntaxError]") -> None:
+        super().__init__(errors)
+        self.errors = errors
+
+    def __str__(self) -> str:
+        return f"Model import failed with {len(self.errors)} errors: " + "; ".join(map(str, self.errors))
+
+
+class CDFAPIException(NeatException):
+    """Raised when there is an error in an API call."""
+
+    def __init__(self, messages: "list[HTTPMessage]") -> None:
+        self.messages = messages
+
+    def __str__(self) -> str:
+        return f"{type(self).__name__}: " + "; ".join(map(str, self.messages))
+
+
+class FileReadException(NeatException):
+    """Raised when there is an error reading a file."""
+
+    def __init__(self, filepath: str, message: str) -> None:
+        super().__init__(f"Error reading file {filepath}: {message}")
+        self.filepath = filepath
+        self.message = message
+
+    def __str__(self) -> str:
+        return f"Error reading file {self.filepath}: {self.message}"
+
+
+class UserInputError(NeatException):
+    """Raised when there is an error in user input."""
+
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.message = message
+
+    def __str__(self) -> str:
+        return f"User input error: {self.message}"

cognite/neat/_issues.py

@@ -0,0 +1,68 @@
+from collections import UserList, defaultdict
+
+from pydantic import BaseModel
+
+
+class Issue(BaseModel):
+    """Base class for all issues"""
+
+    message: str
+    code: str | None = None
+    fix: str | None = None
+
+    @classmethod
+    def issue_type(cls) -> str:
+        return cls.__name__
+
+
+class ModelSyntaxError(Issue):
+    """If any syntax error is found. Stop validation
+    and ask user to fix the syntax error first."""
+
+    ...
+
+
+class ImplementationWarning(Issue):
+    """This is only for conceptual data model. It means that conversion to DMS
+    will fail unless user implements the missing part."""
+
+    ...
+
+
+class ConsistencyError(Issue):
+    """If any consistency error is found, the deployment of the data model will fail. For example,
+    if a reverse direct relations points to a non-existing direct relation. This is only relevant for
+    DMS model.
+    """
+
+    ...
+
+
+class Recommendation(Issue):
+    """Best practice recommendation."""
+
+    ...
+
+
+class IssueList(UserList[Issue]):
+    """A list of issues that can be sorted by type and message."""
+
+    def by_type(self) -> dict[type[Issue], list[Issue]]:
+        """Returns a dictionary of issues sorted by their type."""
+        result: dict[type[Issue], list[Issue]] = defaultdict(list)
+        for issue in self.data:
+            issue_type = type(issue)
+            if issue_type not in result:
+                result[issue_type] = []
+            result[issue_type].append(issue)
+        return result
+
+    def by_code(self) -> dict[str, list[Issue]]:
+        """Returns a dictionary of issues sorted by their code."""
+        result: dict[str, list[Issue]] = defaultdict(list)
+        for issue in self.data:
+            if issue.code is not None:
+                result[issue.code].append(issue)
+            else:
+                result["UNDEFINED"].append(issue)
+        return dict(result)

cognite/neat/_session/__init__.py

@@ -0,0 +1,3 @@
+from ._session import NeatSession
+
+__all__ = ["NeatSession"]

cognite/neat/_session/_html/_render.py

@@ -0,0 +1,30 @@
+from typing import Any, Literal, TypeAlias
+
+from . import static, templates
+
+ENCODING = "utf-8"
+
+Template: TypeAlias = Literal["issues", "deployment"]
+
+
+def render(template_name: Literal["issues", "deployment"], variables: dict[str, Any]) -> str:
+    """Generate HTML content from a template and variables."""
+
+    if template_name not in ["issues", "deployment"]:
+        raise ValueError(f"Unknown template name: {template_name}")
+
+    variables["SHARED_CSS"] = static.shared_style.read_text(encoding=ENCODING)
+
+    if template_name == "issues":
+        template = templates.issues.read_text(encoding=ENCODING)
+        variables["SCRIPTS"] = static.issues_scripts.read_text(encoding=ENCODING)
+        variables["SPECIFIC_CSS"] = static.issues_style.read_text(encoding=ENCODING)
+
+    elif template_name == "deployment":
+        template = templates.deployment.read_text(encoding=ENCODING)
+        variables["SCRIPTS"] = static.deployment_scripts.read_text(encoding=ENCODING)
+        variables["SPECIFIC_CSS"] = static.deployment_style.read_text(encoding=ENCODING)
+
+    for key, value in variables.items():
+        template = template.replace(f"{{{{{key}}}}}", str(value))
+    return template

cognite/neat/_session/_html/static/__init__.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+
+shared_style = Path(__file__).parent / "shared.css"
+issues_style = Path(__file__).parent / "issues.css"
+issues_scripts = Path(__file__).parent / "issues.js"
+
+deployment_style = Path(__file__).parent / "deployment.css"
+deployment_scripts = Path(__file__).parent / "deployment.js"