cognite-neat 1.0.31__py3-none-any.whl → 1.0.33__py3-none-any.whl

cognite/neat/_client/data_classes.py

@@ -2,6 +2,8 @@ from typing import Generic, TypeVar
 
 from pydantic import BaseModel, Field
 
+from cognite.neat._utils.useful_types import BaseModelObject
+
 T = TypeVar("T", bound=BaseModel)
 
 
@@ -42,3 +44,33 @@ class StatisticsResponse(BaseModel, populate_by_name=True):
     concurrent_read_limit: int = Field(alias="concurrentReadLimit")
     concurrent_write_limit: int = Field(alias="concurrentWriteLimit")
     concurrent_delete_limit: int = Field(alias="concurrentDeleteLimit")
+
+
+class SpaceStatisticsItem(BaseModelObject, populate_by_name=True):
+    """Individual space statistics item."""
+
+    space: str
+    containers: int
+    views: int
+    data_models: int
+    edges: int
+    soft_deleted_edges: int
+    nodes: int
+    soft_deleted_nodes: int
+
+    @property
+    def is_empty(self) -> bool:
+        """Check if the space has zero usage."""
+        return (
+            self.containers == 0 and self.views == 0 and self.data_models == 0 and self.edges == 0 and self.nodes == 0
+        )
+
+
+class SpaceStatisticsResponse(BaseModelObject, populate_by_name=True):
+    """Response model for space statistics endpoint."""
+
+    items: list[SpaceStatisticsItem]
+
+    def empty_spaces(self) -> list[str]:
+        """Get a list of space identifiers that have zero usage."""
+        return [item.space for item in self.items if item.is_empty]

cognite/neat/_client/statistics_api.py

@@ -1,7 +1,10 @@
+import json
+
 from cognite.neat._utils.http_client import HTTPClient, ParametersRequest
+from cognite.neat._utils.http_client._data_classes import SimpleBodyRequest
 
 from .config import NeatClientConfig
-from .data_classes import StatisticsResponse
+from .data_classes import SpaceStatisticsResponse, StatisticsResponse
 
 
 class StatisticsAPI:
@@ -27,3 +30,27 @@ class StatisticsAPI:
         result.raise_for_status()
         result = StatisticsResponse.model_validate_json(result.success_response.body)
         return result
+
+    def space_statistics(self, spaces: list[str]) -> SpaceStatisticsResponse:
+        """Retrieve space-wise usage data and limits.
+
+        Args:
+            spaces: List of space identifiers to retrieve statistics for.
+
+        Returns:
+            SpaceStatisticsResponse object.
+        """
+
+        body = {"items": [{"space": space} for space in spaces]}
+
+        result = self._http_client.request_with_retries(
+            SimpleBodyRequest(
+                endpoint_url=self._config.create_api_url("/models/statistics/spaces/byids"),
+                method="POST",
+                body=json.dumps(body),
+            )
+        )
+
+        result.raise_for_status()
+        result = SpaceStatisticsResponse.model_validate_json(result.success_response.body)
+        return result

cognite/neat/_data_model/_analysis.py

@@ -8,6 +8,7 @@ from typing import Literal, TypeAlias, TypeVar
 import networkx as nx
 from pyparsing import cached_property
 
+from cognite.neat._client.data_classes import SpaceStatisticsResponse
 from cognite.neat._data_model._constants import COGNITE_SPACES
 from cognite.neat._data_model._snapshot import SchemaSnapshot
 from cognite.neat._data_model.models.dms._constraints import RequiresConstraintDefinition
@@ -70,9 +71,11 @@ class ValidationResources:
         local: SchemaSnapshot,
         cdf: SchemaSnapshot,
         limits: SchemaLimits | None = None,
+        space_statistics: SpaceStatisticsResponse | None = None,
     ) -> None:
         self._modus_operandi = modus_operandi
         self.limits = limits or SchemaLimits()
+        self.space_statistics = space_statistics
 
         self.local = local
         self.cdf = cdf

cognite/neat/_data_model/_constants.py

@@ -67,8 +67,12 @@ COGNITE_SPACES = (
     "cdf_apm",
     "cdf_apps_shared",
     "cdf_cdm_3d",
+    "cdf_time_series_data",
+    "cdf_cdm_units",
 )
 
+COGNITE_APP_SPACES = ("CommentInstanceSpace", "IndustrialCanvasInstanceSpace", "SolutionTagsInstanceSpace", "scene")
+
 # Defaults from https://docs.cognite.com/cdf/dm/dm_reference/dm_limits_and_restrictions#list-size-limits
 
 DEFAULT_MAX_LIST_SIZE = 1000

cognite/neat/_data_model/deployer/_differ_container.py

@@ -55,7 +55,7 @@ class ContainerDiffer(ItemDiffer[ContainerRequest]):
                 "constraints",
                 current.constraints,
                 new.constraints,
-                add_severity=SeverityType.SAFE,
+                add_severity=SeverityType.WARNING,
                 remove_severity=SeverityType.WARNING,
                 differ=ConstraintDiffer("constraints"),
             )

cognite/neat/_data_model/deployer/data_classes.py

@@ -12,7 +12,6 @@ from pydantic.alias_generators import to_camel
 from cognite.neat._data_model._snapshot import SchemaSnapshot
 from cognite.neat._data_model.models.dms import (
     BaseModelObject,
-    Constraint,
     ContainerConstraintReference,
     ContainerIndexReference,
     ContainerPropertyDefinition,
@@ -20,7 +19,6 @@ from cognite.neat._data_model.models.dms import (
     ContainerRequest,
     DataModelRequest,
     DataModelResource,
-    Index,
     T_DataModelResource,
     T_ResourceId,
     ViewRequest,
@@ -193,7 +191,9 @@ class ContainerDeploymentPlan(ResourceDeploymentPlan[ContainerReference, Contain
     def constraints_to_remove(self) -> dict[ContainerConstraintReference, RemovedField]:
         return self._get_fields_to_remove("constraints.", ContainerConstraintReference)
 
-    def _get_fields_to_remove(self, field_prefix: str, ref_cls: type) -> dict:
+    def _get_fields_to_remove(
+        self, field_prefix: str, ref_cls: type[ContainerIndexReference] | type[ContainerConstraintReference]
+    ) -> dict:
         items: dict = {}
         for resource_change in self.resources:
             for change in resource_change.changes:
@@ -202,7 +202,7 @@ class ContainerDeploymentPlan(ResourceDeploymentPlan[ContainerReference, Contain
                     items[
                         ref_cls(
                             space=resource_change.resource_id.space,
-                            external_id=resource_change.resource_id.external_id,
+                            container_external_id=resource_change.resource_id.external_id,
                             identifier=identifier,
                         )
                     ] = change
@@ -254,7 +254,7 @@ class ResourceDeploymentPlanList(UserList[ResourceDeploymentPlan]):
             updated_resource = resource.model_copy(update={"new_value": resource.current_value})
         elif resource.changes and resource.new_value is not None:
             # Find all field removals and update new_value accordingly.
-            removals = [change for change in resource.changes if isinstance(change, RemovedField)]
+            removals: list[RemovedField] = [change for change in resource.changes if isinstance(change, RemovedField)]
             addition_paths = {change.field_path for change in resource.changes if isinstance(change, AddedField)}
             if removals:
                 if resource.current_value is None:
@@ -271,6 +271,13 @@ class ResourceDeploymentPlanList(UserList[ResourceDeploymentPlan]):
                             for change in resource.changes
                             if not isinstance(change, RemovedField)
                             or (isinstance(change, RemovedField) and change.field_path in addition_paths)
+                            or (
+                                isinstance(change, RemovedField)
+                                and (
+                                    change.field_path.startswith("constraints.")
+                                    or change.field_path.startswith("indexes.")
+                                )
+                            )
                         ],
                     }
                 )
@@ -320,22 +327,14 @@ class ResourceDeploymentPlanList(UserList[ResourceDeploymentPlan]):
         resource: ContainerRequest, removals: list[RemovedField], addition_paths: set[str]
     ) -> DataModelResource:
         container_properties = resource.properties.copy()
-        indexes = (resource.indexes or {}).copy()
-        constraints = (resource.constraints or {}).copy()
         for removal in removals:
             if removal.field_path.startswith("properties."):
                 prop_key = removal.field_path.removeprefix("properties.")
                 container_properties[prop_key] = cast(ContainerPropertyDefinition, removal.current_value)
-            elif removal.field_path.startswith("indexes.") and removal.field_path not in addition_paths:
-                # Index was removed and not re-added, so we need to restore it.
-                index_key = removal.field_path.removeprefix("indexes.")
-                indexes[index_key] = cast(Index, removal.current_value)
-            elif removal.field_path.startswith("constraints.") and removal.field_path not in addition_paths:
-                # Constraint was removed and not re-added, so we need to restore it.
-                constraint_key = removal.field_path.removeprefix("constraints.")
-                constraints[constraint_key] = cast(Constraint, removal.current_value)
+            # Note: indexes and constraints are allowed to be removed in additive mode,
+            # so we don't restore them here unlike properties.
         return resource.model_copy(
-            update={"properties": container_properties, "indexes": indexes or None, "constraints": constraints or None},
+            update={"properties": container_properties},
             deep=True,
         )
 

cognite/neat/_data_model/deployer/deployer.py

@@ -185,16 +185,20 @@ class SchemaDeployer(OnSuccessResultProducer):
                 continue
             current_resource = current_resources[ref]
             diffs = differ.diff(current_resource, new_resource)
-            if (
-                isinstance(current_resource, ContainerRequest)
-                and isinstance(new_resource, ContainerRequest)
-                and self.options.modus_operandi == "additive"
-            ):
-                # In additive mode, changes to constraints and indexes require removal and re-adding
-                # In rebuild mode, all changes are forced via deletion and re-adding
+            if isinstance(current_resource, ContainerRequest) and isinstance(new_resource, ContainerRequest):
+                # CDF doesn't support in-place modification of constraints/indexes,
+                # so we transform changes to remove + add operations in both modes
                 diffs = self.remove_readd_modified_indexes_and_constraints(diffs, current_resource, new_resource)
+
+            warnings = self._generate_warnings_for_constraint_and_index_changes(diffs)
             resources.append(
-                ResourceChange(resource_id=ref, new_value=new_resource, current_value=current_resource, changes=diffs)
+                ResourceChange(
+                    resource_id=ref,
+                    new_value=new_resource,
+                    current_value=current_resource,
+                    changes=diffs,
+                    message=" ".join(warnings) if warnings else None,
+                )
             )
 
         return plan_type(endpoint=endpoint, resources=resources)
@@ -222,19 +226,19 @@ class SchemaDeployer(OnSuccessResultProducer):
                     raise RuntimeError("Bug in Neat. Malformed field path for constraint/index change.")
                 # Field type is either "constraints" or "indexes"
                 field_type, identifier, *_ = diff.field_path.split(".", maxsplit=2)
-                # Mark for removal
+                field_path = f"{field_type}.{identifier}"
                 modified_diffs.append(
                     RemovedField(
-                        field_path=f"{field_type}.{identifier}",
+                        field_path=field_path,
                         item_severity=SeverityType.WARNING,
                         current_value=getattr(current_resource, field_type)[identifier],
                     )
                 )
-                # Mark for addition
+                add_severity = SeverityType.WARNING if field_type == "constraints" else SeverityType.SAFE
                 modified_diffs.append(
                     AddedField(
-                        field_path=f"{field_type}.{identifier}",
-                        item_severity=SeverityType.SAFE,
+                        field_path=field_path,
+                        item_severity=add_severity,
                         new_value=getattr(new_resource, field_type)[identifier],
                     )
                 )
@@ -262,6 +266,29 @@ class SchemaDeployer(OnSuccessResultProducer):
             )
         return None
 
+    @classmethod
+    def _generate_warnings_for_constraint_and_index_changes(cls, diffs: list[FieldChange]) -> list[str]:
+        """Generate warning messages for constraint and index changes.
+
+        Args:
+            diffs: The list of field changes.
+
+        Returns:
+            A list of warning messages for field changes involving constraint and index changes.
+        """
+        warnings: list[str] = []
+        if any(isinstance(diff, AddedField) and diff.field_path.startswith("constraints.") for diff in diffs):
+            warnings.append(
+                "Adding constraints could cause ingestion failures if the data being ingested violates the constraint."
+            )
+        if any(
+            isinstance(diff, RemovedField)
+            and (diff.field_path.startswith("constraints.") or diff.field_path.startswith("indexes."))
+            for diff in diffs
+        ):
+            warnings.append("Removing constraints or indexes may affect query performance.")
+        return warnings
+
     def should_proceed_to_deploy(self, plan: Sequence[ResourceDeploymentPlan]) -> bool:
         max_severity_in_plan = SeverityType.max_severity(
             [change.severity for resource_plan in plan for change in resource_plan.resources],

cognite/neat/_data_model/models/dms/_http.py

@@ -6,6 +6,8 @@ from cognite.neat._utils.useful_types import ReferenceObject
 from ._container import ContainerRequest
 from ._data_model import DataModelRequest
 from ._references import (
+    ContainerConstraintReference,
+    ContainerIndexReference,
     ContainerReference,
     DataModelReference,
     SpaceReference,
@@ -18,7 +20,14 @@ DataModelResource: TypeAlias = SpaceRequest | DataModelRequest | ViewRequest | C
 
 T_DataModelResource = TypeVar("T_DataModelResource", bound=DataModelResource)
 
-ResourceId: TypeAlias = SpaceReference | DataModelReference | ViewReference | ContainerReference
+ResourceId: TypeAlias = (
+    SpaceReference
+    | DataModelReference
+    | ViewReference
+    | ContainerReference
+    | ContainerIndexReference
+    | ContainerConstraintReference
+)
 
 T_ResourceId = TypeVar("T_ResourceId", bound=ResourceId)
 

cognite/neat/_data_model/models/dms/_references.py

@@ -133,9 +133,43 @@ class ViewDirectReference(ReferenceObject):
         return f"{self.source!s}.{self.identifier}"
 
 
-class ContainerIndexReference(ContainerReference):
-    identifier: str
+class ContainerIndexReference(ReferenceObject):
+    """Reference to a container index for deletion API."""
+
+    space: str = Field(
+        description="Id of the space hosting the container.",
+        min_length=1,
+        max_length=43,
+        pattern=SPACE_FORMAT_PATTERN,
+    )
+    container_external_id: str = Field(
+        description="External-id of the container.",
+        min_length=1,
+        max_length=255,
+        pattern=DM_EXTERNAL_ID_PATTERN,
+        alias="containerExternalId",
+    )
+    identifier: str = Field(
+        description="Identifier of the index.",
+    )
 
 
-class ContainerConstraintReference(ContainerReference):
-    identifier: str
+class ContainerConstraintReference(ReferenceObject):
+    """Reference to a container constraint for deletion API."""
+
+    space: str = Field(
+        description="Id of the space hosting the container.",
+        min_length=1,
+        max_length=43,
+        pattern=SPACE_FORMAT_PATTERN,
+    )
+    container_external_id: str = Field(
+        description="External-id of the container.",
+        min_length=1,
+        max_length=255,
+        pattern=DM_EXTERNAL_ID_PATTERN,
+        alias="containerExternalId",
+    )
+    identifier: str = Field(
+        description="Identifier of the constraint.",
+    )

cognite/neat/_data_model/{validation/dms → rules}/_base.py

@@ -2,15 +2,17 @@ from abc import ABC, abstractmethod
 from typing import ClassVar
 
 from cognite.neat._data_model._analysis import ValidationResources
+from cognite.neat._data_model.models.dms._schema import RequestSchema
 from cognite.neat._issues import ConsistencyError, Recommendation
 
 
-class DataModelValidator(ABC):
-    """Assessors for fundamental data model principles."""
+class NeatRule(ABC):
+    """Rules for data model principles."""
 
     code: ClassVar[str]
     issue_type: ClassVar[type[ConsistencyError] | type[Recommendation]]
     alpha: ClassVar[bool] = False
+    fixable: ClassVar[bool] = False
 
     def __init__(
         self,
@@ -19,7 +21,11 @@ class DataModelValidator(ABC):
         self.validation_resources = validation_resources
 
     @abstractmethod
-    def run(self) -> list[ConsistencyError] | list[Recommendation] | list[ConsistencyError | Recommendation]:
-        """Execute the success handler on the data model."""
-        # do something with data model
+    def validate(self) -> list[ConsistencyError] | list[Recommendation] | list[ConsistencyError | Recommendation]:
+        """Execute rule validation."""
         ...
+
+    def fix(self) -> RequestSchema:
+        """Fix the issues found by the validator producing a fixed object."""
+
+        raise NotImplementedError("This rule does not implement fix()")

cognite/neat/_data_model/rules/cdf/__init__.py

@@ -0,0 +1,3 @@
+from ._spaces import EmptySpaces
+
+__all__ = ["EmptySpaces"]

cognite/neat/_data_model/rules/cdf/_base.py

@@ -0,0 +1,5 @@
+from cognite.neat._data_model.rules._base import NeatRule
+
+
+class CDFRule(NeatRule):
+    """Rules for validating entire CDF."""

cognite/neat/_data_model/rules/cdf/_orchestrator.py

@@ -0,0 +1,56 @@
+from collections.abc import Callable
+
+from cognite.neat._client.data_classes import SpaceStatisticsResponse
+from cognite.neat._data_model._analysis import ValidationResources
+from cognite.neat._data_model._shared import OnSuccessIssuesChecker
+from cognite.neat._data_model._snapshot import SchemaSnapshot
+from cognite.neat._data_model.models.dms._limits import SchemaLimits
+from cognite.neat._utils.auxiliary import get_concrete_subclasses
+
+from ._base import CDFRule
+
+
+class CDFRulesOrchestrator(OnSuccessIssuesChecker):
+    """CDF rules orchestrator, used to execute CDF rules on an entire CDF snapshot."""
+
+    def __init__(
+        self,
+        limits: SchemaLimits,
+        space_statistics: SpaceStatisticsResponse,
+        can_run_validator: Callable[[str, type], bool] | None = None,
+        enable_alpha_validators: bool = False,
+    ) -> None:
+        super().__init__()
+        self._limits = limits
+        self._can_run_validator = can_run_validator or (lambda code, issue_type: True)  # type: ignore
+        self._has_run = False
+        self._enable_alpha_validators = enable_alpha_validators
+        self._space_statistics = space_statistics
+
+    def run(self, cdf_snapshot: SchemaSnapshot) -> None:
+        """Run quality assessment on the DMS data model."""
+
+        validation_resources = self._gather_validation_resources(cdf_snapshot)
+
+        # Initialize all validators
+        validators: list[CDFRule] = [validator(validation_resources) for validator in get_concrete_subclasses(CDFRule)]
+
+        # Run validators
+        for validator in validators:
+            if validator.alpha and not self._enable_alpha_validators:
+                continue
+            if self._can_run_validator(validator.code, validator.issue_type):
+                self._issues.extend(validator.validate())
+
+        self._has_run = True
+
+    def _gather_validation_resources(self, cdf_snapshot: SchemaSnapshot) -> ValidationResources:
+        # we do not want to modify the original request schema during validation
+
+        return ValidationResources(
+            cdf=cdf_snapshot,
+            local=cdf_snapshot,
+            limits=self._limits,
+            space_statistics=self._space_statistics,
+            modus_operandi="rebuild",
+        )

cognite/neat/_data_model/rules/cdf/_spaces.py

@@ -0,0 +1,47 @@
+from cognite.neat._data_model._constants import COGNITE_APP_SPACES, COGNITE_SPACES
+from cognite.neat._data_model.rules.cdf._base import CDFRule
+from cognite.neat._issues import Recommendation
+
+BASE_CODE = "NEAT-CDF-SPACES"
+
+
+class EmptySpaces(CDFRule):
+    """Rule that checks for empty spaces in CDF.
+
+    ## What it does
+    This rule checks if there are any empty spaces in CDF.
+
+    ## Why is this bad?
+    CDF projects typically have limits of 100 spaces, and having empty spaces can waste these valuable resources.
+    Also, empty spaces can lead to confusion and mismanagement of resources within the CDF environment.
+    They may indicate incomplete configurations or unused resources that could be cleaned up.
+
+    ## Example
+    A space `iamempty` with no associated resources such as Views, Containers or Data Models.
+
+
+    """
+
+    code = f"{BASE_CODE}-001"
+    issue_type = Recommendation
+
+    def validate(self) -> list[Recommendation]:
+        issues: list[Recommendation] = []
+
+        if not self.validation_resources.space_statistics:
+            return issues
+
+        empty_spaces = set(self.validation_resources.space_statistics.empty_spaces()) - set(
+            COGNITE_APP_SPACES + COGNITE_SPACES
+        )
+
+        for space in empty_spaces:
+            issues.append(
+                Recommendation(
+                    message=f"Space '{space}' is empty and has no associated resources.",
+                    code=self.code,
+                    fix="Consider removing the empty space to maintain a clean CDF environment.",
+                )
+            )
+
+        return issues

cognite/neat/_data_model/{validation → rules}/dms/__init__.py

@@ -36,7 +36,7 @@ from ._limits import (
     ViewImplementsCountIsOutOfLimits,
     ViewPropertyCountIsOutOfLimits,
 )
-from ._orchestrator import DmsDataModelValidation
+from ._orchestrator import DmsDataModelRulesOrchestrator
 from ._performance import (
     MissingRequiresConstraint,
     SuboptimalRequiresConstraint,
@@ -53,7 +53,7 @@ __all__ = [
     "DataModelMissingDescription",
     "DataModelMissingName",
     "DataModelViewCountIsOutOfLimits",
-    "DmsDataModelValidation",
+    "DmsDataModelRulesOrchestrator",
     "EnumerationMissingDescription",
     "EnumerationMissingName",
     "ExternalContainerDoesNotExist",