cognite-neat 0.98.0__py3-none-any.whl → 0.99.1__py3-none-any.whl

cognite/neat/_rules/models/_base_rules.py

@@ -2,8 +2,6 @@
 its sub-models and validators.
 """
 
-from __future__ import annotations
-
 import math
 import sys
 import types
@@ -39,13 +37,16 @@ from rdflib import Namespace, URIRef
 
 from cognite.neat._constants import DEFAULT_NAMESPACE
 from cognite.neat._rules.models._types import (
-    ClassEntityType,
+    ContainerEntityType,
     DataModelExternalIdType,
-    InformationPropertyType,
+    DmsPropertyType,
     SpaceType,
     StrListType,
     VersionType,
+    ViewEntityType,
 )
+from cognite.neat._rules.models.data_types import DataType
+from cognite.neat._rules.models.entities import EdgeEntity, ReverseConnectionEntity, ViewEntity
 
 if sys.version_info >= (3, 11):
     from enum import StrEnum
@@ -249,7 +250,7 @@ class BaseRules(SchemaModel, ABC):
         """Returns a list of headers for the model, typically used by ExcelExporter"""
         headers_by_sheet: dict[str, list[str]] = {}
         for field_name, field in cls.model_fields.items():
-            if field_name == "validators_to_skip":
+            if field_name in ["validators_to_skip", "post_validate"]:
                 continue
             sheet_name = (field.alias or field_name) if by_alias else field_name
             annotation = field.annotation
@@ -379,9 +380,9 @@ class SheetList(list, MutableSequence[T_SheetRow]):
     def __getitem__(self, index: SupportsIndex) -> T_SheetRow: ...
 
     @overload
-    def __getitem__(self, index: slice) -> SheetList[T_SheetRow]: ...
+    def __getitem__(self, index: slice) -> "SheetList[T_SheetRow]": ...
 
-    def __getitem__(self, index: SupportsIndex | slice, /) -> T_SheetRow | SheetList[T_SheetRow]:
+    def __getitem__(self, index: SupportsIndex | slice, /) -> "T_SheetRow | SheetList[T_SheetRow]":
         if isinstance(index, slice):
             return SheetList[T_SheetRow](super().__getitem__(index))
         return super().__getitem__(index)
@@ -399,10 +400,19 @@ ExtensionCategoryType = Annotated[
 
 
 # Immutable such that this can be used as a key in a dictionary
-class PropertyRef(BaseModel, frozen=True):
-    class_: ClassEntityType = Field(alias="Class")
-    property_: InformationPropertyType = Field(alias="Property")
+class ContainerProperty(BaseModel, frozen=True):
+    container: ContainerEntityType
+    property_: DmsPropertyType
+
+
+class ContainerDestinationProperty(ContainerProperty, frozen=True):
+    value_type: DataType | ViewEntity
+    connection: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None = None
+
+
+class ViewRef(BaseModel, frozen=True):
+    view: ViewEntityType
 
 
-class ClassRef(BaseModel, frozen=True):
-    class_: ClassEntityType = Field(alias="Class")
+class ViewProperty(ViewRef, frozen=True):
+    property_: DmsPropertyType

cognite/neat/_rules/models/dms/__init__.py

@@ -1,3 +1,5 @@
+from cognite.neat._client.data_classes.schema import DMSSchema
+
 from ._rules import DMSContainer, DMSEnum, DMSMetadata, DMSNode, DMSProperty, DMSRules, DMSView
 from ._rules_input import (
     DMSInputContainer,
@@ -8,7 +10,7 @@ from ._rules_input import (
     DMSInputRules,
     DMSInputView,
 )
-from ._schema import DMSSchema, PipelineSchema
+from ._validation import DMSValidation
 
 __all__ = [
     "DMSRules",
@@ -19,7 +21,6 @@ __all__ = [
     "DMSContainer",
     "DMSNode",
     "DMSEnum",
-    "PipelineSchema",
     "DMSInputRules",
     "DMSInputMetadata",
     "DMSInputView",
@@ -27,4 +28,5 @@ __all__ = [
     "DMSInputContainer",
     "DMSInputNode",
     "DMSInputEnum",
+    "DMSValidation",
 ]

cognite/neat/_rules/models/dms/_exporter.py

@@ -13,6 +13,14 @@ from cognite.client.data_classes.data_modeling.views import (
     ViewPropertyApply,
 )
 
+from cognite.neat._client.data_classes.data_modeling import (
+    ContainerApplyDict,
+    NodeApplyDict,
+    SpaceApplyDict,
+    ViewApplyDict,
+)
+from cognite.neat._client.data_classes.schema import DMSSchema
+from cognite.neat._constants import COGNITE_SPACES
 from cognite.neat._issues.errors import NeatTypeError, ResourceNotFoundError
 from cognite.neat._issues.warnings import NotSupportedWarning, PropertyNotFoundWarning
 from cognite.neat._issues.warnings.user_modeling import (
@@ -33,10 +41,8 @@ from cognite.neat._rules.models.entities import (
     UnitEntity,
     ViewEntity,
 )
-from cognite.neat._utils.cdf.data_classes import ContainerApplyDict, NodeApplyDict, SpaceApplyDict, ViewApplyDict
 
 from ._rules import DMSEnum, DMSMetadata, DMSProperty, DMSRules, DMSView
-from ._schema import DMSSchema, PipelineSchema
 
 
 class _DMSExporter:
@@ -49,25 +55,13 @@ class _DMSExporter:
         include_pipeline (bool): If True, the pipeline will be included with the schema. Pipeline means the
             raw tables and transformations necessary to populate the data model.
         instance_space (str): The space to use for the instance. Defaults to None,`Rules.metadata.space` will be used
+        remove_cdf_spaces(bool): The
     """
 
-    def __init__(
-        self,
-        rules: DMSRules,
-        include_pipeline: bool = False,
-        instance_space: str | None = None,
-    ):
-        self.include_pipeline = include_pipeline
+    def __init__(self, rules: DMSRules, instance_space: str | None = None, remove_cdf_spaces: bool = False):
         self.instance_space = instance_space
         self.rules = rules
-        self._ref_schema = None
-        if self._ref_schema:
-            # We skip version as that will always be missing in the reference
-            self._ref_views_by_id = {
-                dm.ViewId(view.space, view.external_id): view for view in self._ref_schema.views.values()
-            }
-        else:
-            self._ref_views_by_id = {}  # type: ignore
+        self.remove_cdf_spaces = remove_cdf_spaces
 
     def to_schema(self) -> DMSSchema:
         rules = self.rules
@@ -79,44 +73,39 @@ class _DMSExporter:
             view_properties_by_id, rules.views
         )
 
-        views, view_node_type_filters = self._create_views_with_node_types(
-            view_properties_by_id, view_properties_with_ancestors_by_id
-        )
+        views = self._create_views(view_properties_by_id, view_properties_with_ancestors_by_id)
+        view_node_type_filters: set[dm.NodeId] = set()
+        for dms_view in rules.views:
+            if isinstance(dms_view.filter_, NodeTypeFilter):
+                view_node_type_filters.update(node.as_id() for node in dms_view.filter_.inner or [])
         if rules.nodes:
             node_types = NodeApplyDict(
                 [node.as_node() for node in rules.nodes]
                 + [dm.NodeApply(node.space, node.external_id) for node in view_node_type_filters]
             )
         else:
-            node_types = NodeApplyDict([dm.NodeApply(node.space, node.external_id) for node in view_node_type_filters])
-
-        last_schema: DMSSchema | None = None
+            node_types = NodeApplyDict(
+                [
+                    dm.NodeApply(node.space, node.external_id)
+                    for node in view_node_type_filters
+                    if not (self.remove_cdf_spaces and node.space in COGNITE_SPACES)
+                ]
+            )
 
-        views_not_in_model = {view.view.as_id() for view in rules.views if not view.in_model}
         data_model = rules.metadata.as_data_model()
-
-        data_model_views = [view_id for view_id in views if view_id not in views_not_in_model]
-
         # Sorting to ensure deterministic order
-        data_model.views = sorted(data_model_views, key=lambda v: v.as_tuple())  # type: ignore[union-attr]
-
+        data_model.views = sorted(
+            [dms_view.view.as_id() for dms_view in rules.views if dms_view.in_model],
+            key=lambda x: x.as_tuple(),  # type: ignore[union-attr]
+        )
         spaces = self._create_spaces(rules.metadata, containers, views, data_model)
-
-        output = DMSSchema(
+        return DMSSchema(
             spaces=spaces,
             data_model=data_model,
             views=views,
             containers=containers,
             node_types=node_types,
         )
-        if self.include_pipeline:
-            return PipelineSchema.from_dms(output, self.instance_space)
-
-        if self._ref_schema:
-            output.reference = self._ref_schema
-        if last_schema:
-            output.last = last_schema
-        return output
 
     def _create_spaces(
         self,
@@ -137,14 +126,20 @@ class _DMSExporter:
             spaces[self.instance_space] = dm.SpaceApply(space=self.instance_space, name=self.instance_space)
         return spaces
 
-    def _create_views_with_node_types(
+    def _create_views(
         self,
         view_properties_by_id: dict[dm.ViewId, list[DMSProperty]],
         view_properties_with_ancestors_by_id: dict[dm.ViewId, list[DMSProperty]],
-    ) -> tuple[ViewApplyDict, set[dm.NodeId]]:
+    ) -> ViewApplyDict:
         input_views = list(self.rules.views)
 
-        views = ViewApplyDict([dms_view.as_view() for dms_view in input_views])
+        views = ViewApplyDict(
+            [
+                dms_view.as_view()
+                for dms_view in input_views
+                if not (self.remove_cdf_spaces and dms_view.view.space in COGNITE_SPACES)
+            ]
+        )
 
         for view_id, view in views.items():
             view.properties = {}
@@ -155,11 +150,7 @@ class _DMSExporter:
                 if view_property is not None:
                     view.properties[prop.view_property] = view_property
 
-        unique_node_types: set[dm.NodeId] = set()
-        for view in views.values():
-            unique_node_types.add(dm.NodeId(space=view.space, external_id=view.external_id))
-
-        return views, unique_node_types
+        return views
 
     @classmethod
     def _create_edge_type_from_prop(cls, prop: DMSProperty) -> dm.DirectRelationReference:
@@ -189,7 +180,13 @@ class _DMSExporter:
 
         containers = list(self.rules.containers or [])
 
-        containers = dm.ContainerApplyList([dms_container.as_container() for dms_container in containers])
+        containers = dm.ContainerApplyList(
+            [
+                dms_container.as_container()
+                for dms_container in containers
+                if not (self.remove_cdf_spaces and dms_container.container.space in COGNITE_SPACES)
+            ]
+        )
         container_to_drop = set()
         for container in containers:
             container_id = container.as_id()

cognite/neat/_rules/models/dms/_rules.py

@@ -4,22 +4,26 @@ from typing import Any, ClassVar, Literal
 
 import pandas as pd
 from cognite.client import data_modeling as dm
-from pydantic import Field, field_serializer, field_validator, model_validator
+from pydantic import Field, field_serializer, field_validator
 from pydantic_core.core_schema import SerializationInfo, ValidationInfo
 from rdflib import URIRef
 
+from cognite.neat._client.data_classes.schema import DMSSchema
 from cognite.neat._constants import COGNITE_SPACES
-from cognite.neat._issues import MultiValueError
+from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._issues.warnings import (
     PrincipleMatchingSpaceAndVersionWarning,
 )
 from cognite.neat._rules.models._base_rules import (
     BaseMetadata,
     BaseRules,
+    ContainerProperty,
     DataModelAspect,
     RoleTypes,
     SheetList,
     SheetRow,
+    ViewProperty,
+    ViewRef,
 )
 from cognite.neat._rules.models._types import (
     ClassEntityType,
@@ -44,8 +48,6 @@ from cognite.neat._rules.models.entities import (
     ViewEntityList,
 )
 
-from ._schema import DMSSchema
-
 _DEFAULT_VERSION = "1"
 
 
@@ -180,6 +182,14 @@ class DMSProperty(SheetRow):
             return value.dump(space=metadata.space, version=metadata.version, type=default_type)
         return str(value)
 
+    def as_container_reference(self) -> ContainerProperty:
+        if self.container is None or self.container_property is None:
+            raise NeatValueError("Accessing container reference without container and container property set")
+        return ContainerProperty(container=self.container, property_=self.container_property)
+
+    def as_view_reference(self) -> ViewProperty:
+        return ViewProperty(view=self.view, property_=self.view_property)
+
 
 class DMSContainer(SheetRow):
     container: ContainerEntityType = Field(alias="Container")
@@ -272,10 +282,14 @@ class DMSView(SheetRow):
             version=view_id.version or _DEFAULT_VERSION,
             name=self.name or None,
             description=self.description,
+            filter=None if self.filter_ is None else self.filter_.as_dms_filter(),
             implements=implements,
             properties={},
         )
 
+    def as_view_reference(self) -> ViewRef:
+        return ViewRef(view=self.view)
+
 
 class DMSNode(SheetRow):
     node: DMSNodeEntity = Field(alias="Node")
@@ -356,21 +370,10 @@ class DMSRules(BaseRules):
                 )
         return value
 
-    @model_validator(mode="after")
-    def post_validation(self) -> "DMSRules":
-        from ._validation import DMSPostValidation
-
-        issue_list = DMSPostValidation(self).validate()
-        if issue_list.warnings:
-            issue_list.trigger_warnings()
-        if issue_list.has_errors:
-            raise MultiValueError(issue_list.errors)
-        return self
-
-    def as_schema(self, include_pipeline: bool = False, instance_space: str | None = None) -> DMSSchema:
+    def as_schema(self, instance_space: str | None = None, remove_cdf_spaces: bool = False) -> DMSSchema:
         from ._exporter import _DMSExporter
 
-        return _DMSExporter(self, include_pipeline, instance_space).to_schema()
+        return _DMSExporter(self, instance_space, remove_cdf_spaces=remove_cdf_spaces).to_schema()
 
     def _repr_html_(self) -> str:
         summary = {

cognite/neat/_rules/models/dms/_rules_input.py

@@ -5,6 +5,7 @@ from typing import Any, Literal
 
 import pandas as pd
 from cognite.client import data_modeling as dm
+from cognite.client.data_classes.data_modeling import ContainerId, ViewId
 from rdflib import Namespace, URIRef
 
 from cognite.neat._constants import DEFAULT_NAMESPACE
@@ -125,6 +126,12 @@ class DMSInputProperty(InputComponent[DMSProperty]):
         )
         return output
 
+    def referenced_view(self, default_space: str, default_version: str) -> ViewEntity:
+        return ViewEntity.load(self.view, strict=True, space=default_space, version=default_version)
+
+    def referenced_container(self, default_space: str) -> ContainerEntity | None:
+        return ContainerEntity.load(self.container, strict=True, space=default_space) if self.container else None
+
 
 @dataclass
 class DMSInputContainer(InputComponent[DMSContainer]):
@@ -140,8 +147,7 @@ class DMSInputContainer(InputComponent[DMSContainer]):
 
     def dump(self, default_space: str) -> dict[str, Any]:  # type: ignore[override]
         output = super().dump()
-        container = ContainerEntity.load(self.container, space=default_space)
-        output["Container"] = container
+        output["Container"] = self.as_entity_id(default_space)
         output["Constraint"] = (
             [ContainerEntity.load(constraint.strip(), space=default_space) for constraint in self.constraint.split(",")]
             if self.constraint
@@ -149,6 +155,9 @@ class DMSInputContainer(InputComponent[DMSContainer]):
         )
         return output
 
+    def as_entity_id(self, default_space: str) -> ContainerEntity:
+        return ContainerEntity.load(self.container, strict=True, space=default_space)
+
     @classmethod
     def from_container(cls, container: dm.ContainerApply) -> "DMSInputContainer":
         constraints: list[str] = []
@@ -172,7 +181,7 @@ class DMSInputView(InputComponent[DMSView]):
     name: str | None = None
     description: str | None = None
     implements: str | None = None
-    filter_: Literal["hasData", "nodeType", "rawFilter"] | None = None
+    filter_: Literal["hasData", "nodeType", "rawFilter"] | str | None = None
     in_model: bool = True
     logical: str | None = None
 
@@ -182,17 +191,25 @@ class DMSInputView(InputComponent[DMSView]):
 
     def dump(self, default_space: str, default_version: str) -> dict[str, Any]:  # type: ignore[override]
         output = super().dump()
-        view = ViewEntity.load(self.view, space=default_space, version=default_version)
-        output["View"] = view
-        output["Implements"] = (
+        output["View"] = self.as_entity_id(default_space, default_version)
+        output["Implements"] = self._load_implements(default_space, default_version)
+        return output
+
+    def as_entity_id(self, default_space: str, default_version: str) -> ViewEntity:
+        return ViewEntity.load(self.view, strict=True, space=default_space, version=default_version)
+
+    def _load_implements(self, default_space: str, default_version: str) -> list[ViewEntity] | None:
+        return (
             [
-                ViewEntity.load(implement, space=default_space, version=default_version)
+                ViewEntity.load(implement, strict=True, space=default_space, version=default_version)
                 for implement in self.implements.split(",")
             ]
             if self.implements
             else None
         )
-        return output
+
+    def referenced_views(self, default_space: str, default_version: str) -> list[ViewEntity]:
+        return self._load_implements(default_space, default_version) or []
 
     @classmethod
     def from_view(cls, view: dm.ViewApply, in_model: bool) -> "DMSInputView":
@@ -287,3 +304,30 @@ class DMSInputRules(InputRules[DMSRules]):
         return DEFAULT_NAMESPACE[
             f"data-model/unverified/dms/{self.metadata.space}/{self.metadata.external_id}/{self.metadata.version}"
         ]
+
+    def referenced_views_and_containers(self) -> tuple[set[ViewEntity], set[ContainerEntity]]:
+        default_space = self.metadata.space
+        default_version = self.metadata.version
+
+        containers: set[ContainerEntity] = set()
+        views = {parent for view in self.views for parent in view.referenced_views(default_space, default_version)}
+        for prop in self.properties:
+            views.add(prop.referenced_view(default_space, default_version))
+            if ref_container := prop.referenced_container(default_space):
+                containers.add(ref_container)
+
+        return views, containers
+
+    def as_view_entities(self) -> list[ViewEntity]:
+        return [view.as_entity_id(self.metadata.space, self.metadata.version) for view in self.views]
+
+    def as_container_entities(self) -> list[ContainerEntity]:
+        return [container.as_entity_id(self.metadata.space) for container in self.containers or []]
+
+    def imported_views_and_containers(self) -> tuple[set[ViewEntity], set[ContainerEntity]]:
+        views, containers = self.referenced_views_and_containers()
+        return views - set(self.as_view_entities()), containers - set(self.as_container_entities())
+
+    def imported_views_and_containers_ids(self) -> tuple[set[ViewId], set[ContainerId]]:
+        views, containers = self.imported_views_and_containers()
+        return {view.as_id() for view in views}, {container.as_id() for container in containers}