cognite-neat 0.126.0__py3-none-any.whl → 0.126.1__py3-none-any.whl

cognite/neat/_data_model/models/dms/_views.py

@@ -2,11 +2,11 @@ import re
 from abc import ABC
 from typing import Literal, TypeVar
 
-from pydantic import Field, Json, field_validator, model_validator
+from pydantic import Field, JsonValue, field_validator, model_validator
 
 from cognite.neat._utils.text import humanize_collection
 
-from ._base import Resource, WriteableResource
+from ._base import APIResource, Resource, WriteableResource
 from ._constants import (
     CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN,
     DM_EXTERNAL_ID_PATTERN,
@@ -15,8 +15,9 @@ from ._constants import (
     FORBIDDEN_CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER,
     SPACE_FORMAT_PATTERN,
 )
-from ._references import ContainerReference, ViewReference
+from ._references import ContainerReference, NodeReference, ViewReference
 from ._view_property import (
+    EdgeProperty,
     ViewRequestProperty,
     ViewResponseProperty,
 )
@@ -24,7 +25,7 @@ from ._view_property import (
 KEY_PATTERN = re.compile(CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN)
 
 
-class View(Resource, ABC):
+class View(Resource, APIResource[ViewReference], ABC):
     space: str = Field(
         description="Id of the space that the view belongs to.",
         min_length=1,
@@ -52,7 +53,7 @@ class View(Resource, ABC):
         description="Description of the view.",
         max_length=1024,
     )
-    filter: dict[str, Json] | None = Field(
+    filter: dict[str, JsonValue] | None = Field(
         default=None,
         description="A filter Domain Specific Language (DSL) used to create advanced filter queries.",
     )
@@ -62,7 +63,7 @@ class View(Resource, ABC):
     )
 
     def as_reference(self) -> ViewReference:
-        return ViewReference(space=self.space, externalId=self.external_id, version=self.version)
+        return ViewReference(space=self.space, external_id=self.external_id, version=self.version)
 
     @model_validator(mode="before")
     def set_connection_type_on_primary_properties(cls, data: dict) -> dict:
@@ -134,6 +135,15 @@ class ViewResponse(View, WriteableResource[ViewRequest]):
         """Validate properties Identifier"""
         return _validate_properties_keys(val)
 
+    @property
+    def node_types(self) -> list[NodeReference]:
+        """Get all node types referenced by this view."""
+        nodes_refs: set[NodeReference] = set()
+        for prop in self.properties.values():
+            if isinstance(prop, EdgeProperty):
+                nodes_refs.add(prop.type)
+        return list(nodes_refs)
+
     def as_request(self) -> ViewRequest:
         dumped = self.model_dump(by_alias=True, exclude={"properties"})
         dumped["properties"] = {

cognite/neat/_data_model/validation/_base.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+from typing import ClassVar
+
+from cognite.neat._issues import ConsistencyError, Recommendation
+
+
+class DataModelValidator(ABC):
+    """Assessors for fundamental data model principles."""
+
+    code: ClassVar[str]
+
+    @abstractmethod
+    def run(self) -> list[ConsistencyError] | list[Recommendation]:
+        """Execute the success handler on the data model."""
+        # do something with data model
+        pass

cognite/neat/_data_model/validation/dms/__init__.py

@@ -0,0 +1,9 @@
+from ._orchestrator import DmsDataModelValidation
+from ._validators import UndefinedConnectionEndNodeTypes, VersionSpaceInconsistency, ViewsWithoutProperties
+
+__all__ = [
+    "DmsDataModelValidation",
+    "UndefinedConnectionEndNodeTypes",
+    "VersionSpaceInconsistency",
+    "ViewsWithoutProperties",
+]

cognite/neat/_data_model/validation/dms/_orchestrator.py

@@ -0,0 +1,68 @@
+from cognite.neat._client import NeatClient
+from cognite.neat._data_model._analysis import DataModelAnalysis
+from cognite.neat._data_model._shared import OnSuccessIssuesChecker
+from cognite.neat._data_model.models.dms._references import ViewReference
+from cognite.neat._data_model.models.dms._schema import RequestSchema
+from cognite.neat._data_model.models.dms._views import ViewRequest
+from cognite.neat._data_model.validation._base import DataModelValidator
+
+from ._validators import UndefinedConnectionEndNodeTypes, VersionSpaceInconsistency, ViewsWithoutProperties
+
+
+class DmsDataModelValidation(OnSuccessIssuesChecker):
+    """Placeholder for DMS Quality Assessment functionality."""
+
+    def __init__(
+        self, client: NeatClient | None = None, codes: list[str] | None = None, modus_operandi: str | None = None
+    ) -> None:
+        super().__init__(client)
+        self._codes = codes or ["all"]
+        self._modus_operandi = modus_operandi  # will be used later to trigger how validators will behave
+
+    def run(self, data_model: RequestSchema) -> None:
+        """Run quality assessment on the DMS data model."""
+
+        # Helper wrangled data model components
+        analysis = DataModelAnalysis(data_model)
+        local_views_by_reference = analysis.view_by_reference(include_inherited_properties=True)
+        local_connection_end_node_types = analysis.connection_end_node_types
+        cdf_views_by_reference = self._cdf_view_by_reference(
+            list(analysis.referenced_views(include_connection_end_node_types=True)),
+            include_inherited_properties=True,
+        )
+
+        validators: list[DataModelValidator] = [
+            ViewsWithoutProperties(
+                local_views_by_reference=local_views_by_reference,
+                cdf_views_by_reference=cdf_views_by_reference,
+            ),
+            UndefinedConnectionEndNodeTypes(
+                local_connection_end_node_types=local_connection_end_node_types,
+                local_views_by_reference=local_views_by_reference,
+                cdf_views_by_reference=cdf_views_by_reference,
+            ),
+            VersionSpaceInconsistency(
+                data_model_reference=data_model.data_model.as_reference(),
+                view_references=list(local_views_by_reference.keys()),
+            ),
+        ]
+
+        for validator in validators:
+            if "all" in self._codes or validator.code in self._codes:
+                self._issues.extend(validator.run())
+
+        self._has_run = True
+
+    def _cdf_view_by_reference(
+        self, views: list[ViewReference], include_inherited_properties: bool = True
+    ) -> dict[ViewReference, ViewRequest]:
+        """Fetch view definition from CDF."""
+
+        if not self._client:
+            return {}
+        return {
+            response.as_reference(): response.as_request()
+            for response in self._client.views.retrieve(
+                views, include_inherited_properties=include_inherited_properties
+            )
+        }

cognite/neat/_data_model/validation/dms/_validators.py

@@ -0,0 +1,139 @@
+from cognite.neat._data_model._constants import COGNITE_SPACES
+from cognite.neat._data_model.models.dms._references import DataModelReference, ViewReference
+from cognite.neat._data_model.models.dms._views import ViewRequest
+from cognite.neat._data_model.validation._base import DataModelValidator
+from cognite.neat._issues import ConsistencyError, Recommendation
+
+
+class ViewsWithoutProperties(DataModelValidator):
+    """This validator checks for views without properties, i.e. views that do not have any
+    property attached to them , either directly or through implements."""
+
+    code = "NEAT-DMS-001"
+
+    def __init__(
+        self,
+        local_views_by_reference: dict[ViewReference, ViewRequest],
+        cdf_views_by_reference: dict[ViewReference, ViewRequest],
+    ) -> None:
+        self.local_views_by_reference = local_views_by_reference
+        self.cdf_views_by_reference = cdf_views_by_reference
+
+    def run(self) -> list[ConsistencyError]:
+        views_without_properties = []
+
+        for ref, view in self.local_views_by_reference.items():
+            if not view.properties:
+                # Existing CDF view has properties
+                if (
+                    self.cdf_views_by_reference
+                    and (remote := self.cdf_views_by_reference.get(ref))
+                    and remote.properties
+                ):
+                    continue
+
+                # Implemented views have properties
+                if view.implements and any(
+                    self.cdf_views_by_reference
+                    and (remote_implement := self.cdf_views_by_reference.get(implement))
+                    and remote_implement.properties
+                    for implement in view.implements or []
+                ):
+                    continue
+
+                views_without_properties.append(ref)
+
+        return [
+            ConsistencyError(
+                message=(
+                    f"View {ref!s} does "
+                    "not have any properties defined, either directly or through implements."
+                    " This will prohibit your from deploying the data model to CDF."
+                ),
+                fix="Define properties for the view",
+                code=self.code,
+            )
+            for ref in views_without_properties
+        ]
+
+
+class UndefinedConnectionEndNodeTypes(DataModelValidator):
+    """This validator checks for connections where the end node types are not defined"""
+
+    code = "NEAT-DMS-002"
+
+    def __init__(
+        self,
+        local_connection_end_node_types: dict[tuple[ViewReference, str], ViewReference],
+        local_views_by_reference: dict[ViewReference, ViewRequest],
+        cdf_views_by_reference: dict[ViewReference, ViewRequest],
+    ) -> None:
+        self.local_connection_end_node_types = local_connection_end_node_types
+        self.local_views_by_reference = local_views_by_reference
+        self.cdf_views_by_reference = cdf_views_by_reference
+
+    def run(self) -> list[ConsistencyError]:
+        undefined_value_types = []
+
+        for (view, property_), value_type in self.local_connection_end_node_types.items():
+            if value_type not in self.local_views_by_reference and value_type not in self.cdf_views_by_reference:
+                undefined_value_types.append((view, property_, value_type))
+
+        return [
+            ConsistencyError(
+                message=(
+                    f"View {view!s} property {property_!s} has value type {value_type!s} "
+                    "which is not defined as a view in the data model neither exists in CDF."
+                    " This will prohibit you from deploying the data model to CDF."
+                ),
+                fix="Define necessary view",
+                code=self.code,
+            )
+            for (view, property_, value_type) in undefined_value_types
+        ]
+
+
+class VersionSpaceInconsistency(DataModelValidator):
+    """This validator checks for inconsistencies in versioning and space among views and data model"""
+
+    code = "NEAT-DMS-003"
+
+    def __init__(
+        self,
+        data_model_reference: DataModelReference,
+        view_references: list[ViewReference],
+    ) -> None:
+        self.data_model_reference = data_model_reference
+        self.view_references = view_references
+
+    def run(self) -> list[Recommendation]:
+        recommendations: list[Recommendation] = []
+
+        for view_ref in self.view_references:
+            issue_description = ""
+
+            if view_ref.space not in COGNITE_SPACES:
+                # notify about inconsisten space
+                if view_ref.space != self.data_model_reference.space:
+                    issue_description = f"space (view: {view_ref.space}, data model: {self.data_model_reference.space})"
+
+                # or version if spaces are same
+                elif view_ref.version != self.data_model_reference.version:
+                    issue_description = (
+                        f"version (view: {view_ref.version}, data model: {self.data_model_reference.version})"
+                    )
+
+            if issue_description:
+                recommendations.append(
+                    Recommendation(
+                        message=(
+                            f"View {view_ref!s} has inconsistent {issue_description} "
+                            "with the data model."
+                            " This may lead to more demanding development and maintenance efforts."
+                        ),
+                        fix="Update view version and/or space to match data model",
+                        code=self.code,
+                    )
+                )
+
+        return recommendations

cognite/neat/_exceptions.py

@@ -1,4 +1,8 @@
-from ._issues import ModelSyntaxError
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from cognite.neat._issues import ModelSyntaxError
+    from cognite.neat._utils.http_client import HTTPMessage
 
 
 class NeatException(Exception):
@@ -7,11 +11,19 @@ class NeatException(Exception):
     pass
 
 
-class DataModelImportError(NeatException):
+class DataModelImportException(NeatException):
     """Raised when there is an error importing a model."""
 
-    def __init__(self, errors: list[ModelSyntaxError]) -> None:
+    def __init__(self, errors: "list[ModelSyntaxError]") -> None:
+        super().__init__(errors)
         self.errors = errors
 
     def __str__(self) -> str:
         return f"Model import failed with {len(self.errors)} errors: " + "; ".join(map(str, self.errors))
+
+
+class CDFAPIException(NeatException):
+    """Raised when there is an error in an API call."""
+
+    def __init__(self, messages: "list[HTTPMessage]") -> None:
+        self.messages = messages

cognite/neat/_issues.py

@@ -1,14 +1,23 @@
+from collections import UserList, defaultdict
+
 from pydantic import BaseModel
 
 
-class ModelSyntaxError(BaseModel):
+class Issue(BaseModel):
+    """Base class for all issues"""
+
+    message: str
+    code: str | None = None
+
+
+class ModelSyntaxError(Issue):
     """If any syntax error is found. Stop validation
     and ask user to fix the syntax error first."""
 
-    message: str
+    ...
 
 
-class ImplementationWarning(BaseModel):
+class ImplementationWarning(Issue):
     """This is only for conceptual data model. It means that conversion to DMS
     will fail unless user implements the missing part."""
 
@@ -16,18 +25,42 @@ class ImplementationWarning(BaseModel):
     fix: str
 
 
-class ConsistencyError(BaseModel):
+class ConsistencyError(Issue):
     """If any consistency error is found, the deployment of the data model will fail. For example,
     if a reverse direct relations points to a non-existing direct relation. This is only relevant for
     DMS model.
     """
 
     message: str
-    fix: str
+    fix: str | None = None
 
 
-class Recommendation(BaseModel):
+class Recommendation(Issue):
     """Best practice recommendation."""
 
     message: str
     fix: str | None = None
+
+
+class IssueList(UserList[Issue]):
+    """A list of issues that can be sorted by type and message."""
+
+    def by_type(self) -> dict[type[Issue], list[Issue]]:
+        """Returns a dictionary of issues sorted by their type."""
+        result: dict[type[Issue], list[Issue]] = defaultdict(list)
+        for issue in self.data:
+            issue_type = type(issue)
+            if issue_type not in result:
+                result[issue_type] = []
+            result[issue_type].append(issue)
+        return result
+
+    def by_code(self) -> dict[str, list[Issue]]:
+        """Returns a dictionary of issues sorted by their code."""
+        result: dict[str, list[Issue]] = defaultdict(list)
+        for issue in self.data:
+            if issue.code is not None:
+                result[issue.code].append(issue)
+            else:
+                result["UNDEFINED"].append(issue)
+        return dict(result)

cognite/neat/_session/__init__.py

@@ -0,0 +1,3 @@
+from ._session import NeatSession
+
+__all__ = ["NeatSession"]

cognite/neat/_session/_physical.py

@@ -0,0 +1,88 @@
+from typing import Any
+
+from cognite.neat._client import NeatClient
+from cognite.neat._data_model.exporters import DMSExcelExporter, DMSYamlExporter
+from cognite.neat._data_model.importers import DMSAPIImporter, DMSTableImporter
+from cognite.neat._data_model.models.dms import DataModelReference
+from cognite.neat._data_model.validation.dms import DmsDataModelValidation
+from cognite.neat._store._store import NeatStore
+from cognite.neat._utils._reader import NeatReader
+
+from ._wrappers import session_wrapper
+
+
+class PhysicalDataModel:
+    """Read from a data source into NeatSession graph store."""
+
+    def __init__(self, store: NeatStore, client: NeatClient) -> None:
+        self._store = store
+        self._client = client
+        self.read = ReadPhysicalDataModel(self._store, self._client)
+        self.write = WritePhysicalDataModel(self._store)
+
+
+@session_wrapper
+class ReadPhysicalDataModel:
+    """Read physical data model from various sources into NeatSession graph store."""
+
+    def __init__(self, store: NeatStore, client: NeatClient) -> None:
+        self._store = store
+        self._client = client
+
+    def yaml(self, io: Any) -> None:
+        """Read physical data model from YAML file"""
+
+        path = NeatReader.create(io).materialize_path()
+        reader = DMSTableImporter.from_yaml(path)
+        on_success = DmsDataModelValidation(self._client)
+
+        return self._store.read_physical(reader, on_success)
+
+    def excel(self, io: Any) -> None:
+        """Read physical data model from Excel file"""
+
+        path = NeatReader.create(io).materialize_path()
+        reader = DMSTableImporter.from_excel(path)
+        on_success = DmsDataModelValidation(self._client)
+
+        return self._store.read_physical(reader, on_success)
+
+    def cdf(self, space: str, external_id: str, version: str) -> None:
+        """Read physical data model from CDF
+
+        Args:
+            space (str): The schema space of the data model.
+            external_id (str): The external id of the data model.
+            version (str): The version of the data model.
+
+        """
+        reader = DMSAPIImporter.from_cdf(
+            DataModelReference(space=space, external_id=external_id, version=version), self._client
+        )
+        on_success = DmsDataModelValidation(self._client)
+
+        return self._store.read_physical(reader, on_success)
+
+
+@session_wrapper
+class WritePhysicalDataModel:
+    """Write physical data model to various sources from NeatSession graph store."""
+
+    def __init__(self, store: NeatStore) -> None:
+        self._store = store
+
+    def yaml(self, io: Any) -> None:
+        """Write physical data model to YAML file"""
+
+        file_path = NeatReader.create(io).materialize_path()
+        writer = DMSYamlExporter()
+
+        return self._store.write_physical(writer, file_path=file_path)
+
+    def excel(self, io: Any) -> None:
+        """Write physical data model to Excel file"""
+
+        file_path = NeatReader.create(io).materialize_path()
+        writer = DMSExcelExporter()
+
+        return self._store.write_physical(writer, file_path=file_path)

cognite/neat/_session/_session.py

@@ -1,4 +1,9 @@
-from ._state_machine import EmptyState, ForbiddenState, State
+from cognite.client import ClientConfig, CogniteClient
+
+from cognite.neat._client import NeatClient
+from cognite.neat._store import NeatStore
+
+from ._physical import PhysicalDataModel
 
 
 class NeatSession:
@@ -7,27 +12,31 @@ class NeatSession:
     the state machine for data model and instance operations.
     """
 
-    def __init__(self) -> None:
-        self.state: State = EmptyState()
-
-    def _execute_event(self, event: str) -> bool:
-        """Place holder function for executing events and transitioning states.
-        It will be modified to include actual logic as we progress with v1 of neat.
-
-        """
-        print(f"\n--- Executing event: '{event}' from {self.state} ---")
-
-        old_state = self.state
-        new_state = self.state.on_event(event)
-
-        # Handle ForbiddenState
-        if isinstance(new_state, ForbiddenState):
-            print(f"❌ Event '{event}' is FORBIDDEN from {old_state}")
-            # Return to previous state (as per your table logic)
-            self.state = new_state.on_event("undo")
-            print(f"↩️  Returned to: {self.state}")
-            return False
-        else:
-            self.state = new_state
-            print(f"✅ Transition successful: {old_state} → {self.state}")
-            return True
+    def __init__(self, cognite_client: CogniteClient | ClientConfig) -> None:
+        self._store = NeatStore()
+        self._client = NeatClient(cognite_client)
+        self.physical_data_model = PhysicalDataModel(self._store, self._client)
+        self.issues = Issues(self._store)
+
+
+class Issues:
+    """Class to handle issues in the NeatSession."""
+
+    def __init__(self, store: NeatStore) -> None:
+        self._store = store
+
+    def __call__(self) -> None:
+        if change := self._store.provenance.last_change:
+            if change.errors:
+                print("Critical Issues")
+                for type_, issues in change.errors.by_type().items():
+                    print(f"{type_.__name__}:")
+                    for issue in issues:
+                        print(f"- {issue.message}")
+
+            if change.issues:
+                print("Non-Critical Issues")
+                for type_, issues in change.issues.by_type().items():
+                    print(f"{type_.__name__}:")
+                    for issue in issues:
+                        print(f"- {issue.message}")

cognite/neat/_session/_wrappers.py

@@ -0,0 +1,61 @@
+from collections.abc import Callable
+from functools import wraps
+from typing import Any, Protocol, TypeVar
+
+from cognite.neat._store._store import NeatStore
+from cognite.neat._utils.text import split_on_capitals
+
+
+class HasStore(Protocol):
+    _store: NeatStore
+
+
+T_Class = TypeVar("T_Class", bound=object)
+
+
+def session_wrapper(cls: type[T_Class]) -> type[T_Class]:
+    # 1. Define the method decorator inside
+    def _handle_method_call(func: Callable[..., Any]) -> Callable[..., Any]:
+        """Decorator to handle exceptions and print provenance length"""
+
+        @wraps(func)
+        def wrapper(self: HasStore, *args: Any, **kwargs: Any) -> Any:
+            try:
+                res = func(self, *args, **kwargs)
+                change = self._store.provenance[-1]
+
+                issues_count = len(change.issues) if change.issues else 0
+                errors_count = len(change.errors) if change.errors else 0
+                total_issues = issues_count + errors_count
+
+                newline = "\n"  # python 3.10 compatibility
+                print(
+                    f"{' '.join(split_on_capitals(cls.__name__))} - {func.__name__} "
+                    f"{'✅' if change.successful else '❌'}"
+                    f" | Issues: {total_issues}"
+                    f" (of which {errors_count} critical)"
+                    f"{newline + 'For more details run neat.issues()' if change.issues or change.errors else ''}"
+                )
+
+                return res
+
+            # if an error occurs, we catch it and print it out instead of
+            # getting a full traceback
+            except Exception as e:
+                print(f"{' '.join(split_on_capitals(cls.__name__))} - {func.__name__} ❌")
+                print(f"Error: {e}")
+
+        return wrapper
+
+    # Iterate through all attributes of the class
+    for attr_name in dir(cls):
+        # Skip private/protected methods (starting with _)
+        if not attr_name.startswith("_"):
+            attr = getattr(cls, attr_name)
+            # Only wrap callable methods
+            if callable(attr):
+                # Replace the original method with wrapped version
+                setattr(cls, attr_name, _handle_method_call(attr))
+
+    # Return the modified class
+    return cls

cognite/neat/_state_machine/__init__.py

@@ -0,0 +1,10 @@
+from ._base import State
+from ._states import EmptyState, ForbiddenState, PhysicalState, Undo
+
+__all__ = [
+    "EmptyState",
+    "ForbiddenState",
+    "PhysicalState",
+    "State",
+    "Undo",
+]

cognite/neat/{_session/_state_machine → _state_machine}/_base.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from typing import Any
 
 
 class State(ABC):
@@ -8,12 +9,21 @@ class State(ABC):
         self._store = None
 
     @abstractmethod
-    def on_event(self, event: str) -> "State":
+    def transition(self, event: Any) -> "State":
         """
         Handle events that are delegated to this State.
         """
         raise NotImplementedError("on_event() must be implemented by the subclass.")
 
+    def can_transition(self, event: Any) -> bool:
+        """
+        Check if the state can transition on the given event.
+        """
+        # avoiding circular import
+        from cognite.neat._state_machine._states import ForbiddenState
+
+        return not isinstance(self.transition(event), ForbiddenState)
+
     def __repr__(self) -> str:
         """
         Leverages the __str__ method to describe the State.