cognite-neat 1.0.30__tar.gz → 1.0.32__tar.gz

{cognite_neat-1.0.30 → cognite_neat-1.0.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite-neat
-Version: 1.0.30
+Version: 1.0.32
 Summary: Knowledge graph transformation
 Author: Nikola Vasiljevic, Anders Albert
 Author-email: Nikola Vasiljevic <nikola.vasiljevic@cognite.com>, Anders Albert <anders.albert@cognite.com>

{cognite_neat-1.0.30 → cognite_neat-1.0.32}/cognite/neat/_client/data_classes.py

@@ -2,6 +2,8 @@ from typing import Generic, TypeVar
 
 from pydantic import BaseModel, Field
 
+from cognite.neat._utils.useful_types import BaseModelObject
+
 T = TypeVar("T", bound=BaseModel)
 
 
@@ -42,3 +44,33 @@ class StatisticsResponse(BaseModel, populate_by_name=True):
     concurrent_read_limit: int = Field(alias="concurrentReadLimit")
     concurrent_write_limit: int = Field(alias="concurrentWriteLimit")
     concurrent_delete_limit: int = Field(alias="concurrentDeleteLimit")
+
+
+class SpaceStatisticsItem(BaseModelObject, populate_by_name=True):
+    """Individual space statistics item."""
+
+    space: str
+    containers: int
+    views: int
+    data_models: int
+    edges: int
+    soft_deleted_edges: int
+    nodes: int
+    soft_deleted_nodes: int
+
+    @property
+    def is_empty(self) -> bool:
+        """Check if the space has zero usage."""
+        return (
+            self.containers == 0 and self.views == 0 and self.data_models == 0 and self.edges == 0 and self.nodes == 0
+        )
+
+
+class SpaceStatisticsResponse(BaseModelObject, populate_by_name=True):
+    """Response model for space statistics endpoint."""
+
+    items: list[SpaceStatisticsItem]
+
+    def empty_spaces(self) -> list[str]:
+        """Get a list of space identifiers that have zero usage."""
+        return [item.space for item in self.items if item.is_empty]

cognite_neat-1.0.32/cognite/neat/_client/statistics_api.py

@@ -0,0 +1,56 @@
+import json
+
+from cognite.neat._utils.http_client import HTTPClient, ParametersRequest
+from cognite.neat._utils.http_client._data_classes import SimpleBodyRequest
+
+from .config import NeatClientConfig
+from .data_classes import SpaceStatisticsResponse, StatisticsResponse
+
+
+class StatisticsAPI:
+    def __init__(self, neat_config: NeatClientConfig, http_client: HTTPClient) -> None:
+        self._config = neat_config
+        self._http_client = http_client
+
+    def project(self) -> StatisticsResponse:
+        """Retrieve project-wide usage data and limits.
+
+        Returns:
+            StatisticsResponse object.
+        """
+
+        result = self._http_client.request_with_retries(
+            ParametersRequest(
+                endpoint_url=self._config.create_api_url("/models/statistics"),
+                method="GET",
+                parameters=None,
+            )
+        )
+
+        result.raise_for_status()
+        result = StatisticsResponse.model_validate_json(result.success_response.body)
+        return result
+
+    def space_statistics(self, spaces: list[str]) -> SpaceStatisticsResponse:
+        """Retrieve space-wise usage data and limits.
+
+        Args:
+            spaces: List of space identifiers to retrieve statistics for.
+
+        Returns:
+            SpaceStatisticsResponse object.
+        """
+
+        body = {"items": [{"space": space} for space in spaces]}
+
+        result = self._http_client.request_with_retries(
+            SimpleBodyRequest(
+                endpoint_url=self._config.create_api_url("/models/statistics/spaces/byids"),
+                method="POST",
+                body=json.dumps(body),
+            )
+        )
+
+        result.raise_for_status()
+        result = SpaceStatisticsResponse.model_validate_json(result.success_response.body)
+        return result

{cognite_neat-1.0.30 → cognite_neat-1.0.32}/cognite/neat/_data_model/_analysis.py

@@ -8,6 +8,7 @@ from typing import Literal, TypeAlias, TypeVar
 import networkx as nx
 from pyparsing import cached_property
 
+from cognite.neat._client.data_classes import SpaceStatisticsResponse
 from cognite.neat._data_model._constants import COGNITE_SPACES
 from cognite.neat._data_model._snapshot import SchemaSnapshot
 from cognite.neat._data_model.models.dms._constraints import RequiresConstraintDefinition
@@ -70,9 +71,11 @@ class ValidationResources:
         local: SchemaSnapshot,
         cdf: SchemaSnapshot,
         limits: SchemaLimits | None = None,
+        space_statistics: SpaceStatisticsResponse | None = None,
     ) -> None:
         self._modus_operandi = modus_operandi
         self.limits = limits or SchemaLimits()
+        self.space_statistics = space_statistics
 
         self.local = local
         self.cdf = cdf
@@ -890,10 +893,12 @@ class ValidationResources:
         - to_remove: Existing constraints that are redundant or wrongly oriented
         - status: The optimization status for this view
         """
-        modifiable_containers_in_view = self.containers_by_view.get(view, set()).intersection(
-            self.modifiable_containers
-        )
+        containers_in_view = self.containers_by_view.get(view, set())
+        modifiable_containers_in_view = containers_in_view.intersection(self.modifiable_containers)
         if not modifiable_containers_in_view:
+            # No modifiable containers - check if view is already optimized via existing constraints
+            if self.forms_directed_path(containers_in_view, self.requires_constraint_graph):
+                return RequiresChangesForView(set(), set(), RequiresChangeStatus.OPTIMAL)
             return RequiresChangesForView(set(), set(), RequiresChangeStatus.NO_MODIFIABLE_CONTAINERS)
 
         # Early exit for inherently unsolvable views (no CDM anchor + all modifiables are roots)

{cognite_neat-1.0.30 → cognite_neat-1.0.32}/cognite/neat/_data_model/_constants.py

@@ -67,8 +67,12 @@ COGNITE_SPACES = (
     "cdf_apm",
     "cdf_apps_shared",
     "cdf_cdm_3d",
+    "cdf_time_series_data",
+    "cdf_cdm_units",
 )
 
+COGNITE_APP_SPACES = ("CommentInstanceSpace", "IndustrialCanvasInstanceSpace", "SolutionTagsInstanceSpace", "scene")
+
 # Defaults from https://docs.cognite.com/cdf/dm/dm_reference/dm_limits_and_restrictions#list-size-limits
 
 DEFAULT_MAX_LIST_SIZE = 1000

{cognite_neat-1.0.30/cognite/neat/_data_model/validation/dms → cognite_neat-1.0.32/cognite/neat/_data_model/rules}/_base.py

@@ -2,15 +2,17 @@ from abc import ABC, abstractmethod
 from typing import ClassVar
 
 from cognite.neat._data_model._analysis import ValidationResources
+from cognite.neat._data_model.models.dms._schema import RequestSchema
 from cognite.neat._issues import ConsistencyError, Recommendation
 
 
-class DataModelValidator(ABC):
-    """Assessors for fundamental data model principles."""
+class NeatRule(ABC):
+    """Rules for data model principles."""
 
     code: ClassVar[str]
     issue_type: ClassVar[type[ConsistencyError] | type[Recommendation]]
     alpha: ClassVar[bool] = False
+    fixable: ClassVar[bool] = False
 
     def __init__(
         self,
@@ -19,7 +21,11 @@ class DataModelValidator(ABC):
         self.validation_resources = validation_resources
 
     @abstractmethod
-    def run(self) -> list[ConsistencyError] | list[Recommendation] | list[ConsistencyError | Recommendation]:
-        """Execute the success handler on the data model."""
-        # do something with data model
+    def validate(self) -> list[ConsistencyError] | list[Recommendation] | list[ConsistencyError | Recommendation]:
+        """Execute rule validation."""
         ...
+
+    def fix(self) -> RequestSchema:
+        """Fix the issues found by the validator producing a fixed object."""
+
+        raise NotImplementedError("This rule does not implement fix()")

cognite_neat-1.0.32/cognite/neat/_data_model/rules/cdf/__init__.py

@@ -0,0 +1,3 @@
+from ._spaces import EmptySpaces
+
+__all__ = ["EmptySpaces"]

cognite_neat-1.0.32/cognite/neat/_data_model/rules/cdf/_base.py

@@ -0,0 +1,5 @@
+from cognite.neat._data_model.rules._base import NeatRule
+
+
+class CDFRule(NeatRule):
+    """Rules for validating entire CDF."""

cognite_neat-1.0.32/cognite/neat/_data_model/rules/cdf/_orchestrator.py

@@ -0,0 +1,56 @@
+from collections.abc import Callable
+
+from cognite.neat._client.data_classes import SpaceStatisticsResponse
+from cognite.neat._data_model._analysis import ValidationResources
+from cognite.neat._data_model._shared import OnSuccessIssuesChecker
+from cognite.neat._data_model._snapshot import SchemaSnapshot
+from cognite.neat._data_model.models.dms._limits import SchemaLimits
+from cognite.neat._utils.auxiliary import get_concrete_subclasses
+
+from ._base import CDFRule
+
+
+class CDFRulesOrchestrator(OnSuccessIssuesChecker):
+    """CDF rules orchestrator, used to execute CDF rules on an entire CDF snapshot."""
+
+    def __init__(
+        self,
+        limits: SchemaLimits,
+        space_statistics: SpaceStatisticsResponse,
+        can_run_validator: Callable[[str, type], bool] | None = None,
+        enable_alpha_validators: bool = False,
+    ) -> None:
+        super().__init__()
+        self._limits = limits
+        self._can_run_validator = can_run_validator or (lambda code, issue_type: True)  # type: ignore
+        self._has_run = False
+        self._enable_alpha_validators = enable_alpha_validators
+        self._space_statistics = space_statistics
+
+    def run(self, cdf_snapshot: SchemaSnapshot) -> None:
+        """Run quality assessment on the DMS data model."""
+
+        validation_resources = self._gather_validation_resources(cdf_snapshot)
+
+        # Initialize all validators
+        validators: list[CDFRule] = [validator(validation_resources) for validator in get_concrete_subclasses(CDFRule)]
+
+        # Run validators
+        for validator in validators:
+            if validator.alpha and not self._enable_alpha_validators:
+                continue
+            if self._can_run_validator(validator.code, validator.issue_type):
+                self._issues.extend(validator.validate())
+
+        self._has_run = True
+
+    def _gather_validation_resources(self, cdf_snapshot: SchemaSnapshot) -> ValidationResources:
+        # we do not want to modify the original request schema during validation
+
+        return ValidationResources(
+            cdf=cdf_snapshot,
+            local=cdf_snapshot,
+            limits=self._limits,
+            space_statistics=self._space_statistics,
+            modus_operandi="rebuild",
+        )

cognite_neat-1.0.32/cognite/neat/_data_model/rules/cdf/_spaces.py

@@ -0,0 +1,47 @@
+from cognite.neat._data_model._constants import COGNITE_APP_SPACES, COGNITE_SPACES
+from cognite.neat._data_model.rules.cdf._base import CDFRule
+from cognite.neat._issues import Recommendation
+
+BASE_CODE = "NEAT-CDF-SPACES"
+
+
+class EmptySpaces(CDFRule):
+    """Rule that checks for empty spaces in CDF.
+
+    ## What it does
+    This rule checks if there are any empty spaces in CDF.
+
+    ## Why is this bad?
+    CDF projects typically have limits of 100 spaces, and having empty spaces can waste these valuable resources.
+    Also, empty spaces can lead to confusion and mismanagement of resources within the CDF environment.
+    They may indicate incomplete configurations or unused resources that could be cleaned up.
+
+    ## Example
+    A space `iamempty` with no associated resources such as Views, Containers or Data Models.
+
+
+    """
+
+    code = f"{BASE_CODE}-001"
+    issue_type = Recommendation
+
+    def validate(self) -> list[Recommendation]:
+        issues: list[Recommendation] = []
+
+        if not self.validation_resources.space_statistics:
+            return issues
+
+        empty_spaces = set(self.validation_resources.space_statistics.empty_spaces()) - set(
+            COGNITE_APP_SPACES + COGNITE_SPACES
+        )
+
+        for space in empty_spaces:
+            issues.append(
+                Recommendation(
+                    message=f"Space '{space}' is empty and has no associated resources.",
+                    code=self.code,
+                    fix="Consider removing the empty space to maintain a clean CDF environment.",
+                )
+            )
+
+        return issues

{cognite_neat-1.0.30/cognite/neat/_data_model/validation → cognite_neat-1.0.32/cognite/neat/_data_model/rules}/dms/__init__.py

@@ -36,7 +36,7 @@ from ._limits import (
     ViewImplementsCountIsOutOfLimits,
     ViewPropertyCountIsOutOfLimits,
 )
-from ._orchestrator import DmsDataModelValidation
+from ._orchestrator import DmsDataModelRulesOrchestrator
 from ._performance import (
     MissingRequiresConstraint,
     SuboptimalRequiresConstraint,
@@ -53,7 +53,7 @@ __all__ = [
     "DataModelMissingDescription",
     "DataModelMissingName",
     "DataModelViewCountIsOutOfLimits",
-    "DmsDataModelValidation",
+    "DmsDataModelRulesOrchestrator",
     "EnumerationMissingDescription",
     "EnumerationMissingName",
     "ExternalContainerDoesNotExist",

{cognite_neat-1.0.30/cognite/neat/_data_model/validation → cognite_neat-1.0.32/cognite/neat/_data_model/rules}/dms/_ai_readiness.py

@@ -1,13 +1,13 @@
 """Validators for checking if data model is AI-ready."""
 
 from cognite.neat._data_model.models.dms._data_types import EnumProperty
-from cognite.neat._data_model.validation.dms._base import DataModelValidator
+from cognite.neat._data_model.rules.dms._base import DataModelRule
 from cognite.neat._issues import Recommendation
 
 BASE_CODE = "NEAT-DMS-AI-READINESS"
 
 
-class DataModelMissingName(DataModelValidator):
+class DataModelMissingName(DataModelRule):
     """Validates that data model has a human-readable name.
 
     ## What it does
@@ -26,7 +26,7 @@ class DataModelMissingName(DataModelValidator):
     code = f"{BASE_CODE}-001"
     issue_type = Recommendation
 
-    def run(self) -> list[Recommendation]:
+    def validate(self) -> list[Recommendation]:
         recommendations: list[Recommendation] = []
 
         if not self.validation_resources.merged_data_model.name:
@@ -41,7 +41,7 @@ class DataModelMissingName(DataModelValidator):
         return recommendations
 
 
-class DataModelMissingDescription(DataModelValidator):
+class DataModelMissingDescription(DataModelRule):
     """Validates that data model has a human-readable description.
 
     ## What it does
@@ -65,7 +65,7 @@ class DataModelMissingDescription(DataModelValidator):
     code = f"{BASE_CODE}-002"
     issue_type = Recommendation
 
-    def run(self) -> list[Recommendation]:
+    def validate(self) -> list[Recommendation]:
         recommendations: list[Recommendation] = []
 
         if not self.validation_resources.merged_data_model.description:
@@ -80,7 +80,7 @@ class DataModelMissingDescription(DataModelValidator):
         return recommendations
 
 
-class ViewMissingName(DataModelValidator):
+class ViewMissingName(DataModelRule):
     """Validates that a View has a human-readable name.
 
     ## What it does
@@ -100,7 +100,7 @@ class ViewMissingName(DataModelValidator):
     code = f"{BASE_CODE}-003"
     issue_type = Recommendation
 
-    def run(self) -> list[Recommendation]:
+    def validate(self) -> list[Recommendation]:
         recommendations: list[Recommendation] = []
 
         for view_ref in self.validation_resources.merged_data_model.views or []:
@@ -121,7 +121,7 @@ class ViewMissingName(DataModelValidator):
         return recommendations
 
 
-class ViewMissingDescription(DataModelValidator):
+class ViewMissingDescription(DataModelRule):
     """Validates that a View has a human-readable description.
 
     ## What it does
@@ -151,7 +151,7 @@ class ViewMissingDescription(DataModelValidator):
     code = f"{BASE_CODE}-004"
     issue_type = Recommendation
 
-    def run(self) -> list[Recommendation]:
+    def validate(self) -> list[Recommendation]:
         recommendations: list[Recommendation] = []
 
         for view_ref in self.validation_resources.merged_data_model.views or []:
@@ -172,7 +172,7 @@ class ViewMissingDescription(DataModelValidator):
         return recommendations
 
 
-class ViewPropertyMissingName(DataModelValidator):
+class ViewPropertyMissingName(DataModelRule):
     """Validates that a view property has a human-readable name.
 
     ## What it does
@@ -192,7 +192,7 @@ class ViewPropertyMissingName(DataModelValidator):
     code = f"{BASE_CODE}-005"
     issue_type = Recommendation
 
-    def run(self) -> list[Recommendation]:
+    def validate(self) -> list[Recommendation]:
         recommendations: list[Recommendation] = []
 
         for view_ref in self.validation_resources.merged_data_model.views or []:
@@ -217,7 +217,7 @@ class ViewPropertyMissingName(DataModelValidator):
         return recommendations
 
 
-class ViewPropertyMissingDescription(DataModelValidator):
+class ViewPropertyMissingDescription(DataModelRule):
     """Validates that a View property has a human-readable description.
 
     ## What it does
@@ -248,7 +248,7 @@ class ViewPropertyMissingDescription(DataModelValidator):
     code = f"{BASE_CODE}-006"
     issue_type = Recommendation
 
-    def run(self) -> list[Recommendation]:
+    def validate(self) -> list[Recommendation]:
         recommendations: list[Recommendation] = []
 
         for view_ref in self.validation_resources.merged_data_model.views or []:
@@ -273,7 +273,7 @@ class ViewPropertyMissingDescription(DataModelValidator):
         return recommendations
 
 
-class EnumerationMissingName(DataModelValidator):
+class EnumerationMissingName(DataModelRule):
     """Validates that an enumeration has a human-readable name.
 
     ## What it does
@@ -293,7 +293,7 @@ class EnumerationMissingName(DataModelValidator):
     code = f"{BASE_CODE}-007"
     issue_type = Recommendation
 
-    def run(self) -> list[Recommendation]:
+    def validate(self) -> list[Recommendation]:
         recommendations: list[Recommendation] = []
 
         for container_ref in self.validation_resources.merged.containers:
@@ -322,7 +322,7 @@ class EnumerationMissingName(DataModelValidator):
         return recommendations
 
 
-class EnumerationMissingDescription(DataModelValidator):
+class EnumerationMissingDescription(DataModelRule):
     """Validates that an enumeration value has a human-readable description.
 
     ## What it does
@@ -353,7 +353,7 @@ class EnumerationMissingDescription(DataModelValidator):
     code = f"{BASE_CODE}-008"
     issue_type = Recommendation
 
-    def run(self) -> list[Recommendation]:
+    def validate(self) -> list[Recommendation]:
         recommendations: list[Recommendation] = []
 
         for container_ref in self.validation_resources.merged.containers:

cognite_neat-1.0.32/cognite/neat/_data_model/rules/dms/_base.py

@@ -0,0 +1,5 @@
+from cognite.neat._data_model.rules._base import NeatRule
+
+
+class DataModelRule(NeatRule):
+    """Rules for data model principles."""