cognite-neat 0.70.0__tar.gz → 0.70.2__tar.gz

{cognite_neat-0.70.0 → cognite_neat-0.70.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.70.0
+Version: 0.70.2
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.70.2/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.70.2"

{cognite_neat-0.70.0 → cognite_neat-0.70.2}/cognite/neat/rules/exporters/_rules2dms.py

@@ -1,7 +1,7 @@
 import warnings
 from collections.abc import Collection, Iterable
 from pathlib import Path
-from typing import Literal, TypeAlias
+from typing import Literal, TypeAlias, cast
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes._base import CogniteResourceList
@@ -9,7 +9,8 @@ from cognite.client.exceptions import CogniteAPIError
 
 from cognite.neat.rules._shared import Rules
 from cognite.neat.rules.models._rules import InformationRules
-from cognite.neat.rules.models._rules.dms_architect_rules import DMSRules
+from cognite.neat.rules.models._rules.base import ExtensionCategory, SheetList
+from cognite.neat.rules.models._rules.dms_architect_rules import DMSContainer, DMSRules
 from cognite.neat.rules.models._rules.dms_schema import DMSSchema, PipelineSchema
 from cognite.neat.utils.cdf_loaders import (
     ContainerLoader,
@@ -107,14 +108,53 @@ class DMSExporter(CDFExporter[DMSSchema]):
 
     def export(self, rules: Rules) -> DMSSchema:
         if isinstance(rules, DMSRules):
-            return rules.as_schema(self.standardize_casing, self.export_pipeline, self.instance_space)
+            dms_rules = rules
         elif isinstance(rules, InformationRules):
-            return rules.as_dms_architect_rules().as_schema(
-                self.standardize_casing, self.export_pipeline, self.instance_space
-            )
+            dms_rules = rules.as_dms_architect_rules()
         else:
             raise ValueError(f"{type(rules).__name__} cannot be exported to DMS")
 
+        is_solution_model = (
+            dms_rules.reference and dms_rules.metadata.external_id != dms_rules.reference.metadata.external_id
+        )
+        is_new_model = dms_rules.reference is None
+        if is_new_model or is_solution_model:
+            return dms_rules.as_schema(self.standardize_casing, self.export_pipeline, self.instance_space)
+
+        # This is an extension of an existing model.
+        reference_rules = cast(DMSRules, dms_rules.reference).copy(deep=True)
+        reference_schema = reference_rules.as_schema(self.standardize_casing, self.export_pipeline)
+
+        # Todo Move this to an appropriate location
+        # Merging Reference with User Rules
+        combined_rules = dms_rules.copy(deep=True)
+        existing_containers = {container.class_ for container in combined_rules.containers or []}
+        if combined_rules.containers is None:
+            combined_rules.containers = SheetList[DMSContainer](data=[])
+        for container in reference_rules.containers or []:
+            if container.class_ not in existing_containers:
+                container.reference = None
+                combined_rules.containers.append(container)
+        existing_views = {view.class_ for view in combined_rules.views}
+        for view in reference_rules.views:
+            if view.class_ not in existing_views:
+                view.reference = None
+                combined_rules.views.append(view)
+        existing_properties = {(property_.class_, property_.property_) for property_ in combined_rules.properties}
+        for property_ in reference_rules.properties:
+            if (property_.class_, property_.property_) not in existing_properties:
+                property_.reference = None
+                combined_rules.properties.append(property_)
+
+        schema = combined_rules.as_schema(self.standardize_casing, self.export_pipeline, self.instance_space)
+
+        if dms_rules.metadata.extension in (ExtensionCategory.addition, ExtensionCategory.reshape):
+            # We do not freeze views as they might be changed, even for addition,
+            # in case, for example, new properties are added. The validation will catch this.
+            schema.frozen_ids.update(set(reference_schema.containers.as_ids()))
+            schema.frozen_ids.update(set(reference_schema.node_types.as_ids()))
+        return schema
+
     def export_to_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
         schema = self.export(rules)
         to_export: list[tuple[CogniteResourceList, ResourceLoader]] = []
@@ -135,10 +175,13 @@ class DMSExporter(CDFExporter[DMSSchema]):
         # are changed. This is a workaround to force the conversion.
         redeploy_data_model = False
 
-        for items, loader in to_export:
-            item_ids = loader.get_ids(items)
+        for all_items, loader in to_export:
+            all_item_ids = loader.get_ids(all_items)
+            skipped = sum(1 for item_id in all_item_ids if item_id in schema.frozen_ids)
+            item_ids = [item_id for item_id in all_item_ids if item_id not in schema.frozen_ids]
             cdf_items = loader.retrieve(item_ids)
             cdf_item_by_id = {loader.get_id(item): item for item in cdf_items}
+            items = [item for item in all_items if loader.get_id(item) in item_ids]
             to_create, to_update, unchanged, to_delete = [], [], [], []
             is_redeploying = loader.resource_name == "data_models" and redeploy_data_model
             for item in items:
@@ -163,7 +206,6 @@ class DMSExporter(CDFExporter[DMSSchema]):
             created = len(to_create)
             failed_created = 0
 
-            skipped = 0
             if self.existing_handling in ["update", "force"]:
                 changed = len(to_update)
                 failed_changed = 0

{cognite_neat-0.70.0 → cognite_neat-0.70.2}/cognite/neat/rules/models/_rules/dms_architect_rules.py

@@ -211,7 +211,7 @@ class DMSContainer(SheetEntity):
         container_id = self.container.as_id(default_space, standardize_casing)
         constraints: dict[str, dm.Constraint] = {}
         for constraint in self.constraint or []:
-            requires = dm.RequiresConstraint(constraint.as_id(default_space))
+            requires = dm.RequiresConstraint(constraint.as_id(default_space, standardize_casing))
             constraints[f"{constraint.space}_{constraint.external_id}"] = requires
 
         return dm.ContainerApply(
@@ -632,7 +632,9 @@ class _DMSExporter:
             if not view.in_model
         }
         data_model = rules.metadata.as_data_model()
-        data_model.views = [view_id for view_id in views.as_ids() if view_id not in views_not_in_model]
+        data_model.views = sorted(
+            [view_id for view_id in views.as_ids() if view_id not in views_not_in_model], key=lambda v: v.as_tuple()  # type: ignore[union-attr]
+        )
 
         spaces = self._create_spaces(rules.metadata, containers, views, data_model)
 
@@ -796,7 +798,7 @@ class _DMSExporter:
         parent_views = {parent for view in views for parent in view.implements or []}
         node_type_flag = False
         for view in views:
-            ref_containers = view.referenced_containers()
+            ref_containers = sorted(view.referenced_containers(), key=lambda c: c.as_tuple())
             has_data = dm.filters.HasData(containers=list(ref_containers)) if ref_containers else None
             node_type = dm.filters.Equals(["node", "type"], {"space": view.space, "externalId": view.external_id})
             if view.as_id() in parent_views:

{cognite_neat-0.70.0 → cognite_neat-0.70.2}/cognite/neat/rules/models/_rules/dms_schema.py

@@ -45,6 +45,9 @@ class DMSSchema:
     views: dm.ViewApplyList = field(default_factory=lambda: dm.ViewApplyList([]))
     containers: dm.ContainerApplyList = field(default_factory=lambda: dm.ContainerApplyList([]))
     node_types: dm.NodeApplyList = field(default_factory=lambda: dm.NodeApplyList([]))
+    # The frozen ids are parts of the schema that should not be modified or deleted.
+    # This is used the exporting the schema.
+    frozen_ids: set[dm.ViewId | dm.ContainerId | dm.NodeId] = field(default_factory=set)
 
     _FIELD_NAME_BY_RESOURCE_TYPE: ClassVar[dict[str, str]] = {
         "container": "containers",

{cognite_neat-0.70.0 → cognite_neat-0.70.2}/cognite/neat/rules/models/_rules/information_rules.py

@@ -304,7 +304,7 @@ class InformationRules(RuleModel):
         kwargs = vars(info)
         default_prefix = f"{self.metadata.prefix}:" if self.metadata.prefix else ""
 
-        field_names = ["Class"] if info.by_alias else ["class_"]
+        field_names = ["Class", "Value Type"] if info.by_alias else ["class_", "value_type"]
         properties = []
         for prop in self.properties:
             dumped = prop.model_dump(**kwargs)
@@ -392,7 +392,7 @@ class _InformationRulesConverter:
                 class_=cls_.class_,
                 view=ViewPropEntity(prefix=cls_.class_.prefix, suffix=cls_.class_.suffix, version=cls_.class_.version),
                 description=cls_.description,
-                implements=self._get_view_implements(cls_),
+                implements=self._get_view_implements(cls_, info_metadata),
             )
             for cls_ in self.information.classes
         ]
@@ -507,15 +507,19 @@ class _InformationRulesConverter:
             return ContainerEntity(prefix=prop.class_.prefix, suffix=prop.class_.suffix), prop.property_
 
     @classmethod
-    def _get_view_implements(cls, cls_: InformationClass) -> list[ViewEntity]:
-        if isinstance(cls_.reference, ReferenceEntity):
-            return [
-                ViewPropEntity(
-                    prefix=cls_.reference.prefix, suffix=cls_.reference.suffix, version=cls_.reference.version
-                )
+    def _get_view_implements(cls, cls_: InformationClass, metadata: InformationMetadata) -> list[ViewEntity]:
+        if isinstance(cls_.reference, ReferenceEntity) and cls_.reference.prefix != metadata.prefix:
+            # We use the reference for implements if it is in a different namespace
+            implements = [
+                ViewEntity(prefix=cls_.reference.prefix, suffix=cls_.reference.suffix, version=cls_.reference.version)
             ]
         else:
-            return [
+            implements = []
+
+        implements.extend(
+            [
                 ViewEntity(prefix=parent.prefix, suffix=parent.suffix, version=parent.version)
                 for parent in cls_.parent or []
             ]
+        )
+        return implements

{cognite_neat-0.70.0 → cognite_neat-0.70.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.70.0"
+version = "0.70.2"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.70.0/cognite/neat/_version.py

@@ -1 +0,0 @@
-__version__ = "0.70.0"