cognite-neat 1.0.30__py3-none-any.whl → 1.0.32__py3-none-any.whl

cognite/neat/_data_model/{validation → rules}/dms/_containers.py

@@ -4,13 +4,13 @@ from pyparsing import cast
 
 from cognite.neat._data_model.models.dms._constraints import Constraint, RequiresConstraintDefinition
 from cognite.neat._data_model.models.dms._view_property import ViewCorePropertyRequest
-from cognite.neat._data_model.validation.dms._base import DataModelValidator
+from cognite.neat._data_model.rules.dms._base import DataModelRule
 from cognite.neat._issues import ConsistencyError
 
 BASE_CODE = "NEAT-DMS-CONTAINER"
 
 
-class ExternalContainerDoesNotExist(DataModelValidator):
+class ExternalContainerDoesNotExist(DataModelRule):
     """
     Validates that any container referenced by a view property, when the
     referenced container does not belong to the data model's space, exists in CDF.
@@ -33,7 +33,7 @@ class ExternalContainerDoesNotExist(DataModelValidator):
     code = f"{BASE_CODE}-001"
     issue_type = ConsistencyError
 
-    def run(self) -> list[ConsistencyError]:
+    def validate(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
 
         if not self.validation_resources.merged_data_model.views:
@@ -73,7 +73,7 @@ class ExternalContainerDoesNotExist(DataModelValidator):
         return errors
 
 
-class ExternalContainerPropertyDoesNotExist(DataModelValidator):
+class ExternalContainerPropertyDoesNotExist(DataModelRule):
     """
     Validates that any container property referenced by a view property, when the
     referenced container does not belong to the data model's space, exists in CDF.
@@ -97,7 +97,7 @@ class ExternalContainerPropertyDoesNotExist(DataModelValidator):
     code = f"{BASE_CODE}-002"
     issue_type = ConsistencyError
 
-    def run(self) -> list[ConsistencyError]:
+    def validate(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
 
         if self.validation_resources.merged_data_model.views:
@@ -144,7 +144,7 @@ class ExternalContainerPropertyDoesNotExist(DataModelValidator):
         return errors
 
 
-class RequiredContainerDoesNotExist(DataModelValidator):
+class RequiredContainerDoesNotExist(DataModelRule):
     """
     Validates that any container required by another container exists in the data model.
 
@@ -165,7 +165,7 @@ class RequiredContainerDoesNotExist(DataModelValidator):
     code = f"{BASE_CODE}-003"
     issue_type = ConsistencyError
 
-    def run(self) -> list[ConsistencyError]:
+    def validate(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
 
         for container_ref in self.validation_resources.merged.containers:
@@ -199,7 +199,7 @@ class RequiredContainerDoesNotExist(DataModelValidator):
         return errors
 
 
-class RequiresConstraintCycle(DataModelValidator):
+class RequiresConstraintCycle(DataModelRule):
     """
     Validates that requires constraints between containers do not form cycles.
 
@@ -221,7 +221,7 @@ class RequiresConstraintCycle(DataModelValidator):
     issue_type = ConsistencyError
     alpha = True  # Still in development
 
-    def run(self) -> list[ConsistencyError]:
+    def validate(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
         optimal_edges = self.validation_resources.oriented_mst_edges
 

cognite/neat/_data_model/{validation → rules}/dms/_limits.py

@@ -8,15 +8,15 @@ from cognite.neat._data_model.models.dms._indexes import BtreeIndex, InvertedInd
 from cognite.neat._data_model.models.dms._view_property import (
     ViewCorePropertyRequest,
 )
-from cognite.neat._data_model.validation.dms._base import (
-    DataModelValidator,
+from cognite.neat._data_model.rules.dms._base import (
+    DataModelRule,
 )
 from cognite.neat._issues import ConsistencyError, Recommendation
 
 BASE_CODE = "NEAT-DMS-LIMITS"
 
 
-class DataModelViewCountIsOutOfLimits(DataModelValidator):
+class DataModelViewCountIsOutOfLimits(DataModelRule):
     """Validates that the data model does not exceed the maximum number of views.
 
     ## What it does
@@ -36,7 +36,7 @@ class DataModelViewCountIsOutOfLimits(DataModelValidator):
     code = f"{BASE_CODE}-DATA-MODEL-001"
     issue_type = ConsistencyError
 
-    def run(self) -> list[ConsistencyError]:
+    def validate(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
 
         if self.validation_resources.merged_data_model.views is None:
@@ -66,7 +66,7 @@ class DataModelViewCountIsOutOfLimits(DataModelValidator):
 ### View level limits
 
 
-class ViewPropertyCountIsOutOfLimits(DataModelValidator):
+class ViewPropertyCountIsOutOfLimits(DataModelRule):
     """Validates that a view does not exceed the maximum number of properties.
 
     ## What it does
@@ -83,7 +83,7 @@ class ViewPropertyCountIsOutOfLimits(DataModelValidator):
     code = f"{BASE_CODE}-VIEW-001"
     issue_type = ConsistencyError
 
-    def run(self) -> list[ConsistencyError]:
+    def validate(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
 
         for view_ref in self.validation_resources.merged_data_model.views or []:
@@ -117,7 +117,7 @@ class ViewPropertyCountIsOutOfLimits(DataModelValidator):
         return errors
 
 
-class ViewContainerCountIsOutOfLimits(DataModelValidator):
+class ViewContainerCountIsOutOfLimits(DataModelRule):
     """Validates that a view does not reference too many containers.
 
     ## What it does
@@ -135,7 +135,7 @@ class ViewContainerCountIsOutOfLimits(DataModelValidator):
     code = f"{BASE_CODE}-VIEW-002"
     issue_type = Recommendation
 
-    def run(self) -> list[Recommendation]:
+    def validate(self) -> list[Recommendation]:
         recommendations: list[Recommendation] = []
 
         # Single loop over all views
@@ -166,7 +166,7 @@ class ViewContainerCountIsOutOfLimits(DataModelValidator):
         return recommendations
 
 
-class ViewImplementsCountIsOutOfLimits(DataModelValidator):
+class ViewImplementsCountIsOutOfLimits(DataModelRule):
     """Validates that a view does not implement too many other views.
 
     ## What it does
@@ -183,7 +183,7 @@ class ViewImplementsCountIsOutOfLimits(DataModelValidator):
     code = f"{BASE_CODE}-VIEW-003"
     issue_type = ConsistencyError
 
-    def run(self) -> list[ConsistencyError]:
+    def validate(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
 
         # Single loop over all views
@@ -206,7 +206,7 @@ class ViewImplementsCountIsOutOfLimits(DataModelValidator):
 ### Container level limits
 
 
-class ContainerPropertyCountIsOutOfLimits(DataModelValidator):
+class ContainerPropertyCountIsOutOfLimits(DataModelRule):
     """Validates that a container does not exceed the maximum number of properties.
 
     ## What it does
@@ -224,7 +224,7 @@ class ContainerPropertyCountIsOutOfLimits(DataModelValidator):
     code = f"{BASE_CODE}-CONTAINER-001"
     issue_type = ConsistencyError
 
-    def run(self) -> list[ConsistencyError]:
+    def validate(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
         # Single loop over all containers
         for container_ref in self.validation_resources.merged.containers:
@@ -263,7 +263,7 @@ class ContainerPropertyCountIsOutOfLimits(DataModelValidator):
         return errors
 
 
-class ContainerPropertyListSizeIsOutOfLimits(DataModelValidator):
+class ContainerPropertyListSizeIsOutOfLimits(DataModelRule):
     """Validates that container property list sizes do not exceed CDF limits.
 
     ## What it does
@@ -288,7 +288,7 @@ class ContainerPropertyListSizeIsOutOfLimits(DataModelValidator):
     code = f"{BASE_CODE}-CONTAINER-002"
     issue_type = ConsistencyError
 
-    def run(self) -> list[ConsistencyError]:
+    def validate(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
 
         # Single loop over all containers

cognite/neat/_data_model/{validation → rules}/dms/_orchestrator.py

@@ -6,14 +6,14 @@ from cognite.neat._data_model._shared import OnSuccessIssuesChecker
 from cognite.neat._data_model._snapshot import SchemaSnapshot
 from cognite.neat._data_model.models.dms._limits import SchemaLimits
 from cognite.neat._data_model.models.dms._schema import RequestSchema
+from cognite.neat._data_model.rules.dms._base import DataModelRule
 from cognite.neat._utils.auxiliary import get_concrete_subclasses
 from cognite.neat._utils.useful_types import ModusOperandi
 
-from ._base import DataModelValidator
 
-
-class DmsDataModelValidation(OnSuccessIssuesChecker):
-    """Placeholder for DMS Quality Assessment functionality."""
+class DmsDataModelRulesOrchestrator(OnSuccessIssuesChecker):
+    """DMS Data Model rules orchestrator, used to execute DMS data model rules on a single data model represented
+    as RequestSchema."""
 
     def __init__(
         self,
@@ -37,8 +37,8 @@ class DmsDataModelValidation(OnSuccessIssuesChecker):
         validation_resources = self._gather_validation_resources(request_schema)
 
         # Initialize all validators
-        validators: list[DataModelValidator] = [
-            validator(validation_resources) for validator in get_concrete_subclasses(DataModelValidator)
+        validators: list[DataModelRule] = [
+            validator(validation_resources) for validator in get_concrete_subclasses(DataModelRule)
         ]
 
         # Run validators
@@ -46,7 +46,7 @@ class DmsDataModelValidation(OnSuccessIssuesChecker):
             if validator.alpha and not self._enable_alpha_validators:
                 continue
             if self._can_run_validator(validator.code, validator.issue_type):
-                self._issues.extend(validator.run())
+                self._issues.extend(validator.validate())
 
         self._has_run = True
 

cognite/neat/_data_model/{validation → rules}/dms/_performance.py

@@ -2,13 +2,13 @@
 
 from cognite.neat._data_model._analysis import RequiresChangeStatus
 from cognite.neat._data_model._constants import COGNITE_SPACES
-from cognite.neat._data_model.validation.dms._base import DataModelValidator
+from cognite.neat._data_model.rules.dms._base import DataModelRule
 from cognite.neat._issues import Recommendation
 
 BASE_CODE = "NEAT-DMS-PERFORMANCE"
 
 
-class MissingRequiresConstraint(DataModelValidator):
+class MissingRequiresConstraint(DataModelRule):
     """
     Recommends adding requires constraints to optimize query performance.
 
@@ -31,7 +31,7 @@ class MissingRequiresConstraint(DataModelValidator):
     issue_type = Recommendation
     alpha = True
 
-    def run(self) -> list[Recommendation]:
+    def validate(self) -> list[Recommendation]:
         recommendations: list[Recommendation] = []
 
         for view_ref in self.validation_resources.merged.views:
@@ -50,7 +50,7 @@ class MissingRequiresConstraint(DataModelValidator):
 
                 message = (
                     f"View '{view_ref!s}' is not optimized for querying. "
-                    f"Add a 'requires' constraint from '{src!s}' to '{dst!s}'."
+                    f"Add a 'requires' constraint from the container '{src!s}' to '{dst!s}'."
                 )
                 if not is_safe:
                     # Find a superset view to suggest for ingestion
@@ -75,7 +75,7 @@ class MissingRequiresConstraint(DataModelValidator):
         return recommendations
 
 
-class SuboptimalRequiresConstraint(DataModelValidator):
+class SuboptimalRequiresConstraint(DataModelRule):
     """
     Recommends removing requires constraints that are not optimal.
 
@@ -100,7 +100,7 @@ class SuboptimalRequiresConstraint(DataModelValidator):
     issue_type = Recommendation
     alpha = True
 
-    def run(self) -> list[Recommendation]:
+    def validate(self) -> list[Recommendation]:
         recommendations: list[Recommendation] = []
 
         for view_ref in self.validation_resources.merged.views:
@@ -113,8 +113,9 @@ class SuboptimalRequiresConstraint(DataModelValidator):
                 recommendations.append(
                     Recommendation(
                         message=(
-                            f"View '{view_ref!s}' has a requires constraint '{src!s}' -> '{dst!s}' "
-                            "that is not part of the optimal structure. Consider removing it."
+                            f"View '{view_ref!s}' is mapping to container '{src!s}' "
+                            f"that has a requires constraint to '{dst!s}'. This constraint is "
+                            "not part of the optimal structure. Consider removing it."
                         ),
                         fix="Remove the unnecessary requires constraint",
                         code=self.code,
@@ -124,7 +125,7 @@ class SuboptimalRequiresConstraint(DataModelValidator):
         return recommendations
 
 
-class UnresolvableQueryPerformance(DataModelValidator):
+class UnresolvableQueryPerformance(DataModelRule):
     """
     Identifies views with query performance issues that cannot be resolved.
     This is likely to be caused by unintended modeling choices.
@@ -157,7 +158,7 @@ class UnresolvableQueryPerformance(DataModelValidator):
     issue_type = Recommendation
     alpha = True
 
-    def run(self) -> list[Recommendation]:
+    def validate(self) -> list[Recommendation]:
         recommendations: list[Recommendation] = []
 
         for view_ref in self.validation_resources.merged.views:
@@ -170,8 +171,8 @@ class UnresolvableQueryPerformance(DataModelValidator):
                 recommendations.append(
                     Recommendation(
                         message=(
-                            f"View '{view_ref!s}' has poor query performance. "
-                            "It maps only to CDF built-in containers which cannot have requires constraints. "
+                            f"View '{view_ref!s}' is not optimized for querying. It maps only to CDF "
+                            "built-in containers and some of these lack requires constraints between them. "
                             "Consider adding a view-specific container that requires the others."
                         ),
                         fix="Add a container that requires the others, or restructure the view",
@@ -182,7 +183,7 @@ class UnresolvableQueryPerformance(DataModelValidator):
                 recommendations.append(
                     Recommendation(
                         message=(
-                            f"View '{view_ref!s}' has poor query performance. "
+                            f"View '{view_ref!s}' is not optimized for querying. "
                             "No valid requires constraint solution was found. "
                             "Consider adding a view-specific container that requires the others."
                         ),

cognite/neat/_data_model/{validation → rules}/dms/_views.py

@@ -1,13 +1,13 @@
 """Validators for checking containers in the data model."""
 
 from cognite.neat._data_model.models.dms._view_property import ViewCorePropertyRequest
-from cognite.neat._data_model.validation.dms._base import DataModelValidator
+from cognite.neat._data_model.rules.dms._base import DataModelRule
 from cognite.neat._issues import ConsistencyError
 
 BASE_CODE = "NEAT-DMS-VIEW"
 
 
-class ViewToContainerMappingNotPossible(DataModelValidator):
+class ViewToContainerMappingNotPossible(DataModelRule):
     """Validates that container and container property referenced by view property exist.
 
     ## What it does
@@ -26,7 +26,7 @@ class ViewToContainerMappingNotPossible(DataModelValidator):
     code = f"{BASE_CODE}-001"
     issue_type = ConsistencyError
 
-    def run(self) -> list[ConsistencyError]:
+    def validate(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
 
         if not self.validation_resources.merged_data_model.views:
@@ -79,7 +79,7 @@ class ViewToContainerMappingNotPossible(DataModelValidator):
         return errors
 
 
-class ImplementedViewNotExisting(DataModelValidator):
+class ImplementedViewNotExisting(DataModelRule):
     """Validates that implemented (inherited) view exists.
 
     ## What it does
@@ -97,7 +97,7 @@ class ImplementedViewNotExisting(DataModelValidator):
     code = f"{BASE_CODE}-002"
     issue_type = ConsistencyError
 
-    def run(self) -> list[ConsistencyError]:
+    def validate(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
 
         if not self.validation_resources.merged_data_model.views:
@@ -126,7 +126,7 @@ class ImplementedViewNotExisting(DataModelValidator):
         return errors
 
 
-class CyclicImplements(DataModelValidator):
+class CyclicImplements(DataModelRule):
     """Validates that view implements are not forming a cycle (i.e. cyclic graph of implements)
 
     ## What it does
@@ -146,7 +146,7 @@ class CyclicImplements(DataModelValidator):
     issue_type = ConsistencyError
     alpha = True
 
-    def run(self) -> list[ConsistencyError]:
+    def validate(self) -> list[ConsistencyError]:
         errors: list[ConsistencyError] = []
 
         for cycle in self.validation_resources.implements_cycles:

cognite/neat/_session/_cdf.py

@@ -1,13 +1,15 @@
-# /Users/nikola/repos/neat/cognite/neat/_session/_cdf.py
 import uuid
 
 from cognite.neat._client import NeatClient
 from cognite.neat._config import NeatConfig
+from cognite.neat._data_model.rules.cdf._orchestrator import CDFRulesOrchestrator
+from cognite.neat._session._wrappers import session_wrapper
 from cognite.neat._store._store import NeatStore
 
 from ._html._render import render
 
 
+@session_wrapper
 class CDF:
     """Read from a data source into NeatSession graph store."""
 
@@ -34,3 +36,15 @@ class CDF:
                 "data_models_limit": self._store.cdf_limits.data_models.limit,
             },
         )
+
+    def analyze(self) -> None:
+        """Analyze the entity of CDF data models."""
+
+        on_success = CDFRulesOrchestrator(
+            limits=self._store.cdf_limits,
+            space_statistics=self._store.cdf_space_statistics,
+            can_run_validator=self._config.validation.can_run_validator,
+            enable_alpha_validators=self._config.alpha.enable_experimental_validators,
+        )
+
+        return self._store.cdf_analyze(on_success)

cognite/neat/_session/_physical.py

@@ -16,7 +16,7 @@ from cognite.neat._data_model.exporters import (
 from cognite.neat._data_model.exporters._table_exporter.workbook import WorkbookOptions
 from cognite.neat._data_model.importers import DMSAPICreator, DMSAPIImporter, DMSImporter, DMSTableImporter
 from cognite.neat._data_model.models.dms import DataModelReference
-from cognite.neat._data_model.validation.dms import DmsDataModelValidation
+from cognite.neat._data_model.rules.dms import DmsDataModelRulesOrchestrator
 from cognite.neat._exceptions import UserInputError
 from cognite.neat._state_machine import PhysicalState
 from cognite.neat._store._store import NeatStore
@@ -109,7 +109,7 @@ class ReadPhysicalDataModel:
         else:
             raise UserInputError(f"Unsupported format: {format}. Supported formats are 'neat' and 'toolkit'.")
 
-        on_success = DmsDataModelValidation(
+        on_success = DmsDataModelRulesOrchestrator(
             modus_operandi=self._config.modeling.mode,
             cdf_snapshot=self._store.cdf_snapshot,
             limits=self._store.cdf_limits,
@@ -138,7 +138,7 @@ class ReadPhysicalDataModel:
         else:
             raise UserInputError(f"Unsupported format: {format}. Supported formats are 'neat' and 'toolkit'.")
 
-        on_success = DmsDataModelValidation(
+        on_success = DmsDataModelRulesOrchestrator(
             modus_operandi=self._config.modeling.mode,
             cdf_snapshot=self._store.cdf_snapshot,
             limits=self._store.cdf_limits,
@@ -158,7 +158,7 @@ class ReadPhysicalDataModel:
         path = NeatReader.create(io).materialize_path()
         reader = DMSTableImporter.from_excel(path)
 
-        on_success = DmsDataModelValidation(
+        on_success = DmsDataModelRulesOrchestrator(
             modus_operandi=self._config.modeling.mode,
             cdf_snapshot=self._store.cdf_snapshot,
             limits=self._store.cdf_limits,
@@ -181,7 +181,7 @@ class ReadPhysicalDataModel:
             DataModelReference(space=space, external_id=external_id, version=version), self._client
         )
 
-        on_success = DmsDataModelValidation(
+        on_success = DmsDataModelRulesOrchestrator(
             modus_operandi=self._config.modeling.mode,
             cdf_snapshot=self._store.cdf_snapshot,
             limits=self._store.cdf_limits,
@@ -337,7 +337,7 @@ def create(
         cdf_snapshot=self._store.cdf_snapshot,
     )
 
-    on_success = DmsDataModelValidation(
+    on_success = DmsDataModelRulesOrchestrator(
         modus_operandi=self._config.modeling.mode,
         cdf_snapshot=self._store.cdf_snapshot,
         limits=self._store.cdf_limits,

cognite/neat/_session/_wrappers.py

@@ -28,7 +28,7 @@ def session_wrapper(cls: type[T_Class]) -> type[T_Class]:
             identifier = f"{cls.__name__}.{func.__name__}"
             try:
                 res = func(self, *args, **kwargs)
-                if not self._store.provenance or "DataModel" not in identifier:
+                if not self._store.provenance or ("DataModel" not in identifier and "CDF" not in identifier):
                     print(f"{display_name} ✅")
                     if _COLLECTOR.can_collect:
                         _COLLECTOR.collect("action", {"action": identifier, "success": True})

cognite/neat/_state_machine/_states.py

@@ -34,7 +34,7 @@ class EmptyState(State):
     The initial state with empty NEAT store.
     """
 
-    def transition(self, event: Any) -> State:
+    def transition(self, event: Any | None) -> State:
         if isinstance(event, DMSImporter):
             return PhysicalState()
         return ForbiddenState(self)

cognite/neat/_store/_store.py

@@ -4,6 +4,7 @@ from datetime import datetime, timezone
 from typing import Any, cast
 
 from cognite.neat._client.client import NeatClient
+from cognite.neat._client.data_classes import SpaceStatisticsResponse
 from cognite.neat._config import NeatConfig
 from cognite.neat._data_model._shared import OnSuccess, OnSuccessIssuesChecker, OnSuccessResultProducer
 from cognite.neat._data_model._snapshot import SchemaSnapshot
@@ -36,6 +37,15 @@ class NeatStore:
         # Placeholder for CDF schema and limit snapshot
         self._cdf_snapshot: SchemaSnapshot | None = None
         self._cdf_limits: SchemaLimits | None = None
+        self._cdf_space_statistics: SpaceStatisticsResponse | None = None
+
+    @property
+    def cdf_space_statistics(self) -> SpaceStatisticsResponse:
+        if self._cdf_space_statistics is None:
+            self._cdf_space_statistics = self._client.statistics.space_statistics(
+                [space.space for space in self.cdf_snapshot.spaces.keys()]
+            )
+        return self._cdf_space_statistics
 
     @property
     def cdf_limits(self) -> SchemaLimits:
@@ -91,8 +101,16 @@ class NeatStore:
             and isinstance(on_success, SchemaDeployer)
             and not on_success.options.dry_run
         ):
-            # Update CDF snapshot after successful deployment
+            # Update CDF snapshot and space statistics after deployment
             self._cdf_snapshot = SchemaSnapshot.fetch_entire_cdf(self._client)
+            self._cdf_space_statistics = self._client.statistics.space_statistics(
+                [space.space for space in self.cdf_snapshot.spaces.keys()]
+            )
+
+    def cdf_analyze(self, on_success: OnSuccess) -> None:
+        """Analyze the entity of CDF data models"""
+        change, _ = self._do_activity(lambda: self.cdf_snapshot, on_success)
+        self.provenance.append(change)
 
     def _gather_data_model(self, writer: DMSExporter) -> PhysicalDataModel:
         """Gather the current physical data model from the store

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "1.0.30"
+__version__ = "1.0.32"
 __engine__ = "^2.0.4"

{cognite_neat-1.0.30.dist-info → cognite_neat-1.0.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite-neat
-Version: 1.0.30
+Version: 1.0.32
 Summary: Knowledge graph transformation
 Author: Nikola Vasiljevic, Anders Albert
 Author-email: Nikola Vasiljevic <nikola.vasiljevic@cognite.com>, Anders Albert <anders.albert@cognite.com>