cognite-neat 0.70.0__py3-none-any.whl → 0.70.1__py3-none-any.whl

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.70.0"
+__version__ = "0.70.1"

cognite/neat/rules/exporters/_rules2dms.py

@@ -1,7 +1,7 @@
 import warnings
 from collections.abc import Collection, Iterable
 from pathlib import Path
-from typing import Literal, TypeAlias
+from typing import Literal, TypeAlias, cast
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes._base import CogniteResourceList
@@ -9,7 +9,8 @@ from cognite.client.exceptions import CogniteAPIError
 
 from cognite.neat.rules._shared import Rules
 from cognite.neat.rules.models._rules import InformationRules
-from cognite.neat.rules.models._rules.dms_architect_rules import DMSRules
+from cognite.neat.rules.models._rules.base import ExtensionCategory, SheetList
+from cognite.neat.rules.models._rules.dms_architect_rules import DMSContainer, DMSRules
 from cognite.neat.rules.models._rules.dms_schema import DMSSchema, PipelineSchema
 from cognite.neat.utils.cdf_loaders import (
     ContainerLoader,
@@ -107,14 +108,53 @@ class DMSExporter(CDFExporter[DMSSchema]):
 
     def export(self, rules: Rules) -> DMSSchema:
         if isinstance(rules, DMSRules):
-            return rules.as_schema(self.standardize_casing, self.export_pipeline, self.instance_space)
+            dms_rules = rules
         elif isinstance(rules, InformationRules):
-            return rules.as_dms_architect_rules().as_schema(
-                self.standardize_casing, self.export_pipeline, self.instance_space
-            )
+            dms_rules = rules.as_dms_architect_rules()
         else:
             raise ValueError(f"{type(rules).__name__} cannot be exported to DMS")
 
+        is_solution_model = (
+            dms_rules.reference and dms_rules.metadata.external_id != dms_rules.reference.metadata.external_id
+        )
+        is_new_model = dms_rules.reference is None
+        if is_new_model or is_solution_model:
+            return dms_rules.as_schema(self.standardize_casing, self.export_pipeline, self.instance_space)
+
+        # This is an extension of an existing model.
+        reference_rules = cast(DMSRules, dms_rules.reference).copy(deep=True)
+        reference_schema = reference_rules.as_schema(self.standardize_casing, self.export_pipeline)
+
+        # Todo Move this to an appropriate location
+        # Merging Reference with User Rules
+        combined_rules = dms_rules.copy(deep=True)
+        existing_containers = {container.class_ for container in combined_rules.containers or []}
+        if combined_rules.containers is None:
+            combined_rules.containers = SheetList[DMSContainer](data=[])
+        for container in reference_rules.containers or []:
+            if container.class_ not in existing_containers:
+                container.reference = None
+                combined_rules.containers.append(container)
+        existing_views = {view.class_ for view in combined_rules.views}
+        for view in reference_rules.views:
+            if view.class_ not in existing_views:
+                view.reference = None
+                combined_rules.views.append(view)
+        existing_properties = {(property_.class_, property_.property_) for property_ in combined_rules.properties}
+        for property_ in reference_rules.properties:
+            if (property_.class_, property_.property_) not in existing_properties:
+                property_.reference = None
+                combined_rules.properties.append(property_)
+
+        schema = combined_rules.as_schema(self.standardize_casing, self.export_pipeline, self.instance_space)
+
+        if dms_rules.metadata.extension in (ExtensionCategory.addition, ExtensionCategory.reshape):
+            # We do not freeze views as they might be changed, even for addition,
+            # in case, for example, new properties are added. The validation will catch this.
+            schema.frozen_ids.update(set(reference_schema.containers.as_ids()))
+            schema.frozen_ids.update(set(reference_schema.node_types.as_ids()))
+        return schema
+
     def export_to_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
         schema = self.export(rules)
         to_export: list[tuple[CogniteResourceList, ResourceLoader]] = []
@@ -136,7 +176,9 @@ class DMSExporter(CDFExporter[DMSSchema]):
         redeploy_data_model = False
 
         for items, loader in to_export:
-            item_ids = loader.get_ids(items)
+            all_item_ids = loader.get_ids(items)
+            skipped = sum(1 for item_id in all_item_ids if item_id in schema.frozen_ids)
+            item_ids = [item_id for item_id in all_item_ids if item_id not in schema.frozen_ids]
             cdf_items = loader.retrieve(item_ids)
             cdf_item_by_id = {loader.get_id(item): item for item in cdf_items}
             to_create, to_update, unchanged, to_delete = [], [], [], []
@@ -163,7 +205,6 @@ class DMSExporter(CDFExporter[DMSSchema]):
             created = len(to_create)
             failed_created = 0
 
-            skipped = 0
             if self.existing_handling in ["update", "force"]:
                 changed = len(to_update)
                 failed_changed = 0

cognite/neat/rules/models/_rules/dms_architect_rules.py

@@ -211,7 +211,7 @@ class DMSContainer(SheetEntity):
         container_id = self.container.as_id(default_space, standardize_casing)
         constraints: dict[str, dm.Constraint] = {}
         for constraint in self.constraint or []:
-            requires = dm.RequiresConstraint(constraint.as_id(default_space))
+            requires = dm.RequiresConstraint(constraint.as_id(default_space, standardize_casing))
             constraints[f"{constraint.space}_{constraint.external_id}"] = requires
 
         return dm.ContainerApply(
@@ -796,7 +796,7 @@ class _DMSExporter:
         parent_views = {parent for view in views for parent in view.implements or []}
         node_type_flag = False
         for view in views:
-            ref_containers = view.referenced_containers()
+            ref_containers = sorted(view.referenced_containers(), key=lambda c: c.as_tuple())
             has_data = dm.filters.HasData(containers=list(ref_containers)) if ref_containers else None
             node_type = dm.filters.Equals(["node", "type"], {"space": view.space, "externalId": view.external_id})
             if view.as_id() in parent_views:

cognite/neat/rules/models/_rules/dms_schema.py

@@ -45,6 +45,9 @@ class DMSSchema:
     views: dm.ViewApplyList = field(default_factory=lambda: dm.ViewApplyList([]))
     containers: dm.ContainerApplyList = field(default_factory=lambda: dm.ContainerApplyList([]))
     node_types: dm.NodeApplyList = field(default_factory=lambda: dm.NodeApplyList([]))
+    # The frozen ids are parts of the schema that should not be modified or deleted.
+    # This is used the exporting the schema.
+    frozen_ids: set[dm.ViewId | dm.ContainerId | dm.NodeId] = field(default_factory=set)
 
     _FIELD_NAME_BY_RESOURCE_TYPE: ClassVar[dict[str, str]] = {
         "container": "containers",

cognite/neat/rules/models/_rules/information_rules.py

@@ -304,7 +304,7 @@ class InformationRules(RuleModel):
         kwargs = vars(info)
         default_prefix = f"{self.metadata.prefix}:" if self.metadata.prefix else ""
 
-        field_names = ["Class"] if info.by_alias else ["class_"]
+        field_names = ["Class", "Value Type"] if info.by_alias else ["class_", "value_type"]
         properties = []
         for prop in self.properties:
             dumped = prop.model_dump(**kwargs)
@@ -392,7 +392,7 @@ class _InformationRulesConverter:
                 class_=cls_.class_,
                 view=ViewPropEntity(prefix=cls_.class_.prefix, suffix=cls_.class_.suffix, version=cls_.class_.version),
                 description=cls_.description,
-                implements=self._get_view_implements(cls_),
+                implements=self._get_view_implements(cls_, info_metadata),
             )
             for cls_ in self.information.classes
         ]
@@ -507,15 +507,19 @@ class _InformationRulesConverter:
             return ContainerEntity(prefix=prop.class_.prefix, suffix=prop.class_.suffix), prop.property_
 
     @classmethod
-    def _get_view_implements(cls, cls_: InformationClass) -> list[ViewEntity]:
-        if isinstance(cls_.reference, ReferenceEntity):
-            return [
-                ViewPropEntity(
-                    prefix=cls_.reference.prefix, suffix=cls_.reference.suffix, version=cls_.reference.version
-                )
+    def _get_view_implements(cls, cls_: InformationClass, metadata: InformationMetadata) -> list[ViewEntity]:
+        if isinstance(cls_.reference, ReferenceEntity) and cls_.reference.prefix != metadata.prefix:
+            # We use the reference for implements if it is in a different namespace
+            implements = [
+                ViewEntity(prefix=cls_.reference.prefix, suffix=cls_.reference.suffix, version=cls_.reference.version)
             ]
         else:
-            return [
+            implements = []
+
+        implements.extend(
+            [
                 ViewEntity(prefix=parent.prefix, suffix=parent.suffix, version=parent.version)
                 for parent in cls_.parent or []
             ]
+        )
+        return implements

{cognite_neat-0.70.0.dist-info → cognite_neat-0.70.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.70.0
+Version: 0.70.1
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

{cognite_neat-0.70.0.dist-info → cognite_neat-0.70.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 cognite/neat/__init__.py,sha256=v-rRiDOgZ3sQSMQKq0vgUQZvpeOkoHFXissAx6Ktg84,61
-cognite/neat/_version.py,sha256=3IhgUqYnqMysoIwxT7jN8hqTm0UVFXntZR74E9Nb9vI,23
+cognite/neat/_version.py,sha256=WVjSRSd2zgGQFanscfNpVjo_q_ShT12r7ukn4JBk-AU,23
 cognite/neat/app/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/app/api/asgi/metrics.py,sha256=nxFy7L5cChTI0a-zkCiJ59Aq8yLuIJp5c9Dg0wRXtV0,152
 cognite/neat/app/api/configuration.py,sha256=xnKdBE_dtq1nRvKa79YGA_wimI5UhoSRuBQz4LkLzQw,4606
@@ -113,7 +113,7 @@ cognite/neat/rules/exporter/_validation.py,sha256=CnEer181-EdnWl3yOuZJKOJ80W7H7q
 cognite/neat/rules/exporters/__init__.py,sha256=Gn3CjkVKHJF9Po1ZPH4wAJ-sRW9up7b2CpXm-eReV3Q,413
 cognite/neat/rules/exporters/_base.py,sha256=aRCzRjGDoXILkGvASev2UjVvLXrVCRTFiKgYwzJZqAo,1518
 cognite/neat/rules/exporters/_models.py,sha256=f_RbFhoyD27z6Dsk4upYMHp7JHZRCfIvZsEzH5JSvtc,1645
-cognite/neat/rules/exporters/_rules2dms.py,sha256=R39FpCNx7l5-A2yheR-IPT5fHDviq-6OxyqF9UC2yt4,9688
+cognite/neat/rules/exporters/_rules2dms.py,sha256=5KUVBYyGr4Q4YVqBs6taTeh8nDVM1L-ABGFTpcaph4c,12129
 cognite/neat/rules/exporters/_rules2excel.py,sha256=uK-aS9sBwXUCFQLIO1jciBQrB4w7ySF6MafDqG23hrQ,9080
 cognite/neat/rules/exporters/_rules2ontology.py,sha256=70T28d1tcT179kt5yOZQlLgAQ67uydz3XbhbA10IRuI,19768
 cognite/neat/rules/exporters/_rules2yaml.py,sha256=HKgFlpgD7U2vWLQC_VdSvkDk19QgeVAd_J8wQ9ZgrN8,3038
@@ -163,10 +163,10 @@ cognite/neat/rules/models/_rules/_types/_base.py,sha256=EsTqjXUP0zpuQ-Z_BMP-00rJ
 cognite/neat/rules/models/_rules/_types/_field.py,sha256=dOVAU1jWCupFVnrYYwLfI-nNUC4rv4vXHMzpiObtWiw,10295
 cognite/neat/rules/models/_rules/_types/_value.py,sha256=I7Ke388M2NOi0U5F9kAzOjE0O4ZXbIxefupgDAjnISM,6079
 cognite/neat/rules/models/_rules/base.py,sha256=9DgtdCmpz84sMFxZB_stWkalVbjA4HQKsTMpSjjOVLU,10635
-cognite/neat/rules/models/_rules/dms_architect_rules.py,sha256=K0A8iJN8FRfiZQO-jhCBii1EGcALtgth9VZbo56PSWQ,49323
-cognite/neat/rules/models/_rules/dms_schema.py,sha256=KmxMTa9Iircc_3sUGMg2zQkSHjxLmtAwaXxzssjfAWQ,30010
+cognite/neat/rules/models/_rules/dms_architect_rules.py,sha256=aevTRWHscgYBih4DqdMzKPAHpj5NVOrCpld0sTp-5Qk,49379
+cognite/neat/rules/models/_rules/dms_schema.py,sha256=-ru40beGY2WJvf9_sd5eO2Wh8x2qLQ2UmgzExloBWac,30229
 cognite/neat/rules/models/_rules/domain_rules.py,sha256=mOE4M6wOurmnAehxbnxvP9vIVXsFuKSiyMmD1shXKpA,2051
-cognite/neat/rules/models/_rules/information_rules.py,sha256=9Wjhpq5itp1CblRMFYKckFzeQDUp8kkArFzbEEK_lUA,20732
+cognite/neat/rules/models/_rules/information_rules.py,sha256=e_SesgvMbzgC9WMFx0b_6qrpOIQvoQ_kXUiXrSYC9RI,20984
 cognite/neat/rules/models/raw_rules.py,sha256=Y7ZVKyvLhX0s6WdwztbFiYFj4EijGEfjtBmv0ibRmmg,12368
 cognite/neat/rules/models/rdfpath.py,sha256=kk1dqxl3n2W_vSQtpSJri2O4nCXnCDZB2lhLsLV1PN0,7344
 cognite/neat/rules/models/rules.py,sha256=ECKrQEtoiQqtR8W-rKMAWmODpHeHhI8Qzf3TwCa3dy8,51063
@@ -227,8 +227,8 @@ cognite/neat/workflows/steps_registry.py,sha256=PZVoHX4d6Vmjz6XzUFnFFWMCnrVnqkUC
 cognite/neat/workflows/tasks.py,sha256=dqlJwKAb0jlkl7abbY8RRz3m7MT4SK8-7cntMWkOYjw,788
 cognite/neat/workflows/triggers.py,sha256=_BLNplzoz0iic367u1mhHMHiUrCwP-SLK6_CZzfODX0,7071
 cognite/neat/workflows/utils.py,sha256=gKdy3RLG7ctRhbCRwaDIWpL9Mi98zm56-d4jfHDqP1E,453
-cognite_neat-0.70.0.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-0.70.0.dist-info/METADATA,sha256=3SfpZKbevbpJ1NnjVPu_CA3rlMNF_fPMoRKj1JFGRMs,9321
-cognite_neat-0.70.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cognite_neat-0.70.0.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
-cognite_neat-0.70.0.dist-info/RECORD,,
+cognite_neat-0.70.1.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-0.70.1.dist-info/METADATA,sha256=olXq5x4vGn839BRr4LQBKuE8gsBpqDuiUW57iH9he1k,9321
+cognite_neat-0.70.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cognite_neat-0.70.1.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
+cognite_neat-0.70.1.dist-info/RECORD,,