cognite-neat 0.123.26__py3-none-any.whl → 1.0.22__py3-none-any.whl

cognite/neat/_data_model/validation/dms/_orchestrator.py

@@ -0,0 +1,70 @@
+from collections.abc import Callable
+from datetime import datetime, timezone
+
+from cognite.neat._data_model._analysis import ValidationResources
+from cognite.neat._data_model._shared import OnSuccessIssuesChecker
+from cognite.neat._data_model._snapshot import SchemaSnapshot
+from cognite.neat._data_model.models.dms._limits import SchemaLimits
+from cognite.neat._data_model.models.dms._schema import RequestSchema
+from cognite.neat._utils.auxiliary import get_concrete_subclasses
+from cognite.neat._utils.useful_types import ModusOperandi
+
+from ._base import DataModelValidator
+
+
+class DmsDataModelValidation(OnSuccessIssuesChecker):
+    """Placeholder for DMS Quality Assessment functionality."""
+
+    def __init__(
+        self,
+        cdf_snapshot: SchemaSnapshot,
+        limits: SchemaLimits,
+        modus_operandi: ModusOperandi = "additive",
+        can_run_validator: Callable[[str, type], bool] | None = None,
+        enable_alpha_validators: bool = False,
+    ) -> None:
+        super().__init__()
+        self._cdf_snapshot = cdf_snapshot
+        self._limits = limits
+        self._modus_operandi = modus_operandi
+        self._can_run_validator = can_run_validator or (lambda code, issue_type: True)  # type: ignore
+        self._has_run = False
+        self._enable_alpha_validators = enable_alpha_validators
+
+    def run(self, request_schema: RequestSchema) -> None:
+        """Run quality assessment on the DMS data model."""
+
+        validation_resources = self._gather_validation_resources(request_schema)
+
+        # Initialize all validators
+        validators: list[DataModelValidator] = [
+            validator(validation_resources) for validator in get_concrete_subclasses(DataModelValidator)
+        ]
+
+        # Run validators
+        for validator in validators:
+            if validator.alpha and not self._enable_alpha_validators:
+                continue
+            if self._can_run_validator(validator.code, validator.issue_type):
+                self._issues.extend(validator.run())
+
+        self._has_run = True
+
+    def _gather_validation_resources(self, request_schema: RequestSchema) -> ValidationResources:
+        # we do not want to modify the original request schema during validation
+        copy = request_schema.model_copy(deep=True)
+        local = SchemaSnapshot(
+            data_model={request_schema.data_model.as_reference(): copy.data_model},
+            views={view.as_reference(): view for view in copy.views},
+            containers={container.as_reference(): container for container in copy.containers},
+            spaces={space.as_reference(): space for space in copy.spaces},
+            node_types={node_type: node_type for node_type in copy.node_types},
+            timestamp=datetime.now(timezone.utc),
+        )
+
+        return ValidationResources(
+            cdf=self._cdf_snapshot,
+            local=local,
+            limits=self._limits,
+            modus_operandi=self._modus_operandi,
+        )

cognite/neat/_data_model/validation/dms/_views.py

@@ -0,0 +1,164 @@
+"""Validators for checking containers in the data model."""
+
+from cognite.neat._data_model.models.dms._view_property import ViewCorePropertyRequest
+from cognite.neat._data_model.validation.dms._base import DataModelValidator
+from cognite.neat._issues import ConsistencyError
+
+BASE_CODE = "NEAT-DMS-VIEW"
+
+
+class ViewToContainerMappingNotPossible(DataModelValidator):
+    """Validates that container and container property referenced by view property exist.
+
+    ## What it does
+    Validates that for each view property that maps to a container and container property,
+    the referenced container and container property exist.
+
+    ## Why is this bad?
+    If a view property references a container or container property that does not exist,
+    the data model cannot be deployed to CDF. This means that view property will not be able to function.
+
+    ## Example
+    View WindTurbine has property location that maps to container WindTurbineContainer and property gpsCoordinates.
+    If WindTurbineContainer and/or property gpsCoordinates does not exist, the data model cannot be deployed to CDF.
+    """
+
+    code = f"{BASE_CODE}-001"
+    issue_type = ConsistencyError
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+
+        if not self.validation_resources.merged_data_model.views:
+            return errors
+
+        for view_ref in self.validation_resources.merged_data_model.views:
+            view = self.validation_resources.select_view(view_ref)
+
+            if not view:
+                raise RuntimeError(
+                    f"{type(self).__name__}: View {view_ref!s} not found in local resources. This is a bug in NEAT."
+                )
+
+            if view.properties is None:
+                continue
+
+            for property_ref, property_ in view.properties.items():
+                if not isinstance(property_, ViewCorePropertyRequest):
+                    continue
+
+                container_ref = property_.container
+                container_property = property_.container_property_identifier
+
+                container = self.validation_resources.select_container(container_ref, container_property)
+
+                if not container:
+                    errors.append(
+                        ConsistencyError(
+                            message=(
+                                f"View {view_ref!s} property {property_ref!s} maps to "
+                                f"container {container_ref!s} which does not exist."
+                            ),
+                            fix="Define necessary container",
+                            code=self.code,
+                        )
+                    )
+                elif container_property not in container.properties:
+                    errors.append(
+                        ConsistencyError(
+                            message=(
+                                f"View {view_ref!s} property {property_ref!s} maps to "
+                                f"container {container_ref!s} which does not have "
+                                f"property '{container_property}'."
+                            ),
+                            fix="Define necessary container property",
+                            code=self.code,
+                        )
+                    )
+
+        return errors
+
+
+class ImplementedViewNotExisting(DataModelValidator):
+    """Validates that implemented (inherited) view exists.
+
+    ## What it does
+    Validates that all views which are implemented (inherited) in the data model actually exist either locally
+    or in CDF.
+
+    ## Why is this bad?
+    If a view being implemented (inherited) does not exist, the data model cannot be deployed to CDF.
+
+    ## Example
+    If view WindTurbine implements (inherits) view Asset, but Asset view does not exist in the data model
+    or in CDF, the data model cannot be deployed to CDF.
+    """
+
+    code = f"{BASE_CODE}-002"
+    issue_type = ConsistencyError
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+
+        if not self.validation_resources.merged_data_model.views:
+            return errors
+
+        for view_ref in self.validation_resources.merged_data_model.views:
+            view = self.validation_resources.select_view(view_ref)
+
+            if not view:
+                raise RuntimeError(
+                    f"{type(self).__name__}: View {view_ref!s} not found in local resources. This is a bug in NEAT."
+                )
+
+            if view.implements is None:
+                continue
+            for implement in view.implements:
+                if self.validation_resources.select_view(implement) is None:
+                    errors.append(
+                        ConsistencyError(
+                            message=f"View {view_ref!s} implements {implement!s} which is not defined.",
+                            fix="Define the missing view or remove it from the implemented views list",
+                            code=self.code,
+                        )
+                    )
+
+        return errors
+
+
+class CyclicImplements(DataModelValidator):
+    """Validates that view implements are not forming a cycle (i.e. cyclic graph of implements)
+
+    ## What it does
+    Runs graph analysis of the implements graph finding every cycle within the graph
+
+    ## Why is this bad?
+    You will not be able to deploy the data model to CDF, since cyclic implements are impossible to resolve
+    in terms of inheritance of properties.
+
+    ## Example
+    Say we have following views: A, B, and C, where A implements B, B implements C, and C implements A. This forms
+    the cyclic graph of implements A->B->C->A.
+
+    """
+
+    code = f"{BASE_CODE}-003"
+    issue_type = ConsistencyError
+    alpha = True
+
+    def run(self) -> list[ConsistencyError]:
+        errors: list[ConsistencyError] = []
+
+        for cycle in self.validation_resources.implements_cycles:
+            errors.append(
+                ConsistencyError(
+                    message=(
+                        "Detected cycle of view implements "
+                        f"{'->'.join([str(view) for view in cycle]) + f'->{cycle[0]}'}"
+                    ),
+                    fix="Inspect involved views and make necessary fix",
+                    code=self.code,
+                )
+            )
+
+        return errors

cognite/neat/_exceptions.py

@@ -0,0 +1,68 @@
+from abc import ABC
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from cognite.neat._issues import ModelSyntaxError
+    from cognite.neat._utils.http_client import HTTPMessage
+
+
+class NeatException(Exception, ABC):
+    """Base class for all exceptions raised by Neat."""
+
+    pass
+
+
+class DataModelImportException(NeatException):
+    """Raised when there is an error importing a model."""
+
+    def __init__(self, errors: "list[ModelSyntaxError]") -> None:
+        super().__init__(errors)
+        self.errors = errors
+
+    def __str__(self) -> str:
+        return f"Model import failed with {len(self.errors)} errors: " + "; ".join(map(str, self.errors))
+
+
+class CDFAPIException(NeatException):
+    """Raised when there is an error in an API call."""
+
+    def __init__(self, messages: "list[HTTPMessage]") -> None:
+        self.messages = messages
+
+    def __str__(self) -> str:
+        return f"{type(self).__name__}: " + "; ".join(map(str, self.messages))
+
+
+class FileReadException(NeatException):
+    """Raised when there is an error reading a file."""
+
+    def __init__(self, filepath: str, message: str) -> None:
+        super().__init__(f"Error reading file {filepath}: {message}")
+        self.filepath = filepath
+        self.message = message
+
+    def __str__(self) -> str:
+        return f"Error reading file {self.filepath}: {self.message}"
+
+
+class UserInputError(NeatException):
+    """Raised when there is an error in user input."""
+
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.message = message
+
+    def __str__(self) -> str:
+        return f"User input error: {self.message}"
+
+
+class NeatImportError(NeatException, ImportError):
+    def __init__(self, module: str, neat_extra: str, functionality: str = "functionality") -> None:
+        message = (
+            f"The {functionality} requires {module}. You can include it in your neat "
+            f'installation with `pip install "cognite-neat[{neat_extra}]"`.'
+        )
+        super().__init__(message)
+
+    def __str__(self) -> str:
+        return self.args[0]

cognite/neat/_issues.py

@@ -0,0 +1,68 @@
+from collections import UserList, defaultdict
+
+from pydantic import BaseModel
+
+
+class Issue(BaseModel):
+    """Base class for all issues"""
+
+    message: str
+    code: str | None = None
+    fix: str | None = None
+
+    @classmethod
+    def issue_type(cls) -> str:
+        return cls.__name__
+
+
+class ModelSyntaxError(Issue):
+    """If any syntax error is found. Stop validation
+    and ask user to fix the syntax error first."""
+
+    ...
+
+
+class ImplementationWarning(Issue):
+    """This is only for conceptual data model. It means that conversion to DMS
+    will fail unless user implements the missing part."""
+
+    ...
+
+
+class ConsistencyError(Issue):
+    """If any consistency error is found, the deployment of the data model will fail. For example,
+    if a reverse direct relations points to a non-existing direct relation. This is only relevant for
+    DMS model.
+    """
+
+    ...
+
+
+class Recommendation(Issue):
+    """Best practice recommendation."""
+
+    ...
+
+
+class IssueList(UserList[Issue]):
+    """A list of issues that can be sorted by type and message."""
+
+    def by_type(self) -> dict[type[Issue], list[Issue]]:
+        """Returns a dictionary of issues sorted by their type."""
+        result: dict[type[Issue], list[Issue]] = defaultdict(list)
+        for issue in self.data:
+            issue_type = type(issue)
+            if issue_type not in result:
+                result[issue_type] = []
+            result[issue_type].append(issue)
+        return result
+
+    def by_code(self) -> dict[str, list[Issue]]:
+        """Returns a dictionary of issues sorted by their code."""
+        result: dict[str, list[Issue]] = defaultdict(list)
+        for issue in self.data:
+            if issue.code is not None:
+                result[issue.code].append(issue)
+            else:
+                result["UNDEFINED"].append(issue)
+        return dict(result)

cognite/neat/_session/__init__.py

@@ -0,0 +1,3 @@
+from ._session import NeatSession
+
+__all__ = ["NeatSession"]

cognite/neat/_session/_html/_render.py

@@ -0,0 +1,30 @@
+from typing import Any, Literal, TypeAlias
+
+from . import static, templates
+
+ENCODING = "utf-8"
+
+Template: TypeAlias = Literal["issues", "deployment"]
+
+
+def render(template_name: Literal["issues", "deployment"], variables: dict[str, Any]) -> str:
+    """Generate HTML content from a template and variables."""
+
+    if template_name not in ["issues", "deployment"]:
+        raise ValueError(f"Unknown template name: {template_name}")
+
+    variables["SHARED_CSS"] = static.shared_style.read_text(encoding=ENCODING)
+
+    if template_name == "issues":
+        template = templates.issues.read_text(encoding=ENCODING)
+        variables["SCRIPTS"] = static.issues_scripts.read_text(encoding=ENCODING)
+        variables["SPECIFIC_CSS"] = static.issues_style.read_text(encoding=ENCODING)
+
+    elif template_name == "deployment":
+        template = templates.deployment.read_text(encoding=ENCODING)
+        variables["SCRIPTS"] = static.deployment_scripts.read_text(encoding=ENCODING)
+        variables["SPECIFIC_CSS"] = static.deployment_style.read_text(encoding=ENCODING)
+
+    for key, value in variables.items():
+        template = template.replace(f"{{{{{key}}}}}", str(value))
+    return template

cognite/neat/_session/_html/static/__init__.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+
+shared_style = Path(__file__).parent / "shared.css"
+issues_style = Path(__file__).parent / "issues.css"
+issues_scripts = Path(__file__).parent / "issues.js"
+
+deployment_style = Path(__file__).parent / "deployment.css"
+deployment_scripts = Path(__file__).parent / "deployment.js"