cognite-neat 0.99.0__py3-none-any.whl → 0.99.1__py3-none-any.whl

cognite/neat/_rules/models/dms/_exporter.py

@@ -20,6 +20,7 @@ from cognite.neat._client.data_classes.data_modeling import (
     ViewApplyDict,
 )
 from cognite.neat._client.data_classes.schema import DMSSchema
+from cognite.neat._constants import COGNITE_SPACES
 from cognite.neat._issues.errors import NeatTypeError, ResourceNotFoundError
 from cognite.neat._issues.warnings import NotSupportedWarning, PropertyNotFoundWarning
 from cognite.neat._issues.warnings.user_modeling import (
@@ -54,19 +55,13 @@ class _DMSExporter:
         include_pipeline (bool): If True, the pipeline will be included with the schema. Pipeline means the
             raw tables and transformations necessary to populate the data model.
         instance_space (str): The space to use for the instance. Defaults to None,`Rules.metadata.space` will be used
+        remove_cdf_spaces(bool): The
     """
 
-    def __init__(self, rules: DMSRules, instance_space: str | None = None):
+    def __init__(self, rules: DMSRules, instance_space: str | None = None, remove_cdf_spaces: bool = False):
         self.instance_space = instance_space
         self.rules = rules
-        self._ref_schema = None
-        if self._ref_schema:
-            # We skip version as that will always be missing in the reference
-            self._ref_views_by_id = {
-                dm.ViewId(view.space, view.external_id): view for view in self._ref_schema.views.values()
-            }
-        else:
-            self._ref_views_by_id = {}  # type: ignore
+        self.remove_cdf_spaces = remove_cdf_spaces
 
     def to_schema(self) -> DMSSchema:
         rules = self.rules
@@ -78,7 +73,7 @@ class _DMSExporter:
             view_properties_by_id, rules.views
         )
 
-        views = self._create_views_with_node_types(view_properties_by_id, view_properties_with_ancestors_by_id)
+        views = self._create_views(view_properties_by_id, view_properties_with_ancestors_by_id)
         view_node_type_filters: set[dm.NodeId] = set()
         for dms_view in rules.views:
             if isinstance(dms_view.filter_, NodeTypeFilter):
@@ -89,21 +84,22 @@ class _DMSExporter:
                 + [dm.NodeApply(node.space, node.external_id) for node in view_node_type_filters]
             )
         else:
-            node_types = NodeApplyDict([dm.NodeApply(node.space, node.external_id) for node in view_node_type_filters])
-
-        last_schema: DMSSchema | None = None
+            node_types = NodeApplyDict(
+                [
+                    dm.NodeApply(node.space, node.external_id)
+                    for node in view_node_type_filters
+                    if not (self.remove_cdf_spaces and node.space in COGNITE_SPACES)
+                ]
+            )
 
-        views_not_in_model = {view.view.as_id() for view in rules.views if not view.in_model}
         data_model = rules.metadata.as_data_model()
-
-        data_model_views = [view_id for view_id in views if view_id not in views_not_in_model]
-
         # Sorting to ensure deterministic order
-        data_model.views = sorted(data_model_views, key=lambda v: v.as_tuple())  # type: ignore[union-attr]
-
+        data_model.views = sorted(
+            [dms_view.view.as_id() for dms_view in rules.views if dms_view.in_model],
+            key=lambda x: x.as_tuple(),  # type: ignore[union-attr]
+        )
         spaces = self._create_spaces(rules.metadata, containers, views, data_model)
-
-        output = DMSSchema(
+        return DMSSchema(
             spaces=spaces,
             data_model=data_model,
             views=views,
@@ -111,12 +107,6 @@ class _DMSExporter:
             node_types=node_types,
         )
 
-        if self._ref_schema:
-            output.reference = self._ref_schema
-        if last_schema:
-            output.last = last_schema
-        return output
-
     def _create_spaces(
         self,
         metadata: DMSMetadata,
@@ -136,14 +126,20 @@ class _DMSExporter:
             spaces[self.instance_space] = dm.SpaceApply(space=self.instance_space, name=self.instance_space)
         return spaces
 
-    def _create_views_with_node_types(
+    def _create_views(
         self,
         view_properties_by_id: dict[dm.ViewId, list[DMSProperty]],
         view_properties_with_ancestors_by_id: dict[dm.ViewId, list[DMSProperty]],
     ) -> ViewApplyDict:
         input_views = list(self.rules.views)
 
-        views = ViewApplyDict([dms_view.as_view() for dms_view in input_views])
+        views = ViewApplyDict(
+            [
+                dms_view.as_view()
+                for dms_view in input_views
+                if not (self.remove_cdf_spaces and dms_view.view.space in COGNITE_SPACES)
+            ]
+        )
 
         for view_id, view in views.items():
             view.properties = {}
@@ -184,7 +180,13 @@ class _DMSExporter:
 
         containers = list(self.rules.containers or [])
 
-        containers = dm.ContainerApplyList([dms_container.as_container() for dms_container in containers])
+        containers = dm.ContainerApplyList(
+            [
+                dms_container.as_container()
+                for dms_container in containers
+                if not (self.remove_cdf_spaces and dms_container.container.space in COGNITE_SPACES)
+            ]
+        )
         container_to_drop = set()
         for container in containers:
             container_id = container.as_id()

cognite/neat/_rules/models/dms/_rules.py

@@ -4,13 +4,12 @@ from typing import Any, ClassVar, Literal
 
 import pandas as pd
 from cognite.client import data_modeling as dm
-from pydantic import Field, field_serializer, field_validator, model_validator
+from pydantic import Field, field_serializer, field_validator
 from pydantic_core.core_schema import SerializationInfo, ValidationInfo
 from rdflib import URIRef
 
 from cognite.neat._client.data_classes.schema import DMSSchema
 from cognite.neat._constants import COGNITE_SPACES
-from cognite.neat._issues import MultiValueError
 from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._issues.warnings import (
     PrincipleMatchingSpaceAndVersionWarning,
@@ -339,9 +338,6 @@ class DMSRules(BaseRules):
     containers: SheetList[DMSContainer] | None = Field(None, alias="Containers")
     enum: SheetList[DMSEnum] | None = Field(None, alias="Enum")
     nodes: SheetList[DMSNode] | None = Field(None, alias="Nodes")
-    # This is a hack to allow the post_validation to be turned off when needed
-    # Will likely be moved completely out of the rules in the future
-    post_validate: bool = Field(default=True, exclude=True, repr=False)
 
     @field_validator("views")
     def matching_version_and_space(cls, value: SheetList[DMSView], info: ValidationInfo) -> SheetList[DMSView]:
@@ -374,23 +370,10 @@ class DMSRules(BaseRules):
                 )
         return value
 
-    @model_validator(mode="after")
-    def post_validation(self) -> "DMSRules":
-        from ._validation import DMSPostValidation
-
-        if not self.post_validate:
-            return self
-        issue_list = DMSPostValidation(self).validate()
-        if issue_list.warnings:
-            issue_list.trigger_warnings()
-        if issue_list.has_errors:
-            raise MultiValueError(issue_list.errors)
-        return self
-
-    def as_schema(self, include_pipeline: bool = False, instance_space: str | None = None) -> DMSSchema:
+    def as_schema(self, instance_space: str | None = None, remove_cdf_spaces: bool = False) -> DMSSchema:
         from ._exporter import _DMSExporter
 
-        return _DMSExporter(self, instance_space).to_schema()
+        return _DMSExporter(self, instance_space, remove_cdf_spaces=remove_cdf_spaces).to_schema()
 
     def _repr_html_(self) -> str:
         summary = {
@@ -406,28 +389,3 @@ class DMSRules(BaseRules):
         }
 
         return pd.DataFrame([summary]).T.rename(columns={0: ""})._repr_html_()  # type: ignore
-
-    def imported_views_and_containers_ids(
-        self, include_model_views_with_no_properties: bool = True
-    ) -> tuple[set[dm.ViewId], set[dm.ContainerId]]:
-        existing_views = {view.view for view in self.views}
-        imported_views: set[dm.ViewId] = set()
-        for view in self.views:
-            for parent in view.implements or []:
-                if parent not in existing_views:
-                    imported_views.add(parent.as_id())
-        existing_containers = {container.container for container in self.containers or []}
-        imported_containers: set[dm.ContainerId] = set()
-        view_with_properties: set[ViewEntity] = set()
-        for prop in self.properties:
-            if prop.container and prop.container not in existing_containers:
-                imported_containers.add(prop.container.as_id())
-            if prop.view not in existing_views:
-                imported_views.add(prop.view.as_id())
-            view_with_properties.add(prop.view)
-
-        if include_model_views_with_no_properties:
-            extra_views = existing_views - view_with_properties
-            imported_views.update({view.as_id() for view in extra_views})
-
-        return imported_views, imported_containers