cognite-neat 0.96.3__tar.gz → 0.96.5__tar.gz

{cognite_neat-0.96.3 → cognite_neat-0.96.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.96.3
+Version: 0.96.5
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

{cognite_neat-0.96.3 → cognite_neat-0.96.5}/cognite/neat/_issues/_base.py

@@ -109,7 +109,11 @@ class NeatIssue:
     def dump(self) -> dict[str, Any]:
         """Return a dictionary representation of the issue."""
         variables = vars(self)
-        output = {to_camel(key): self._dump_value(value) for key, value in variables.items() if value is not None}
+        output = {
+            to_camel(key): self._dump_value(value)
+            for key, value in variables.items()
+            if not (value is None or key.startswith("_"))
+        }
         output["NeatIssue"] = type(self).__name__
         return output
 

{cognite_neat-0.96.3 → cognite_neat-0.96.5}/cognite/neat/_rules/exporters/_base.py

@@ -32,9 +32,11 @@ class BaseExporter(ABC, Generic[T_VerifiedRules, T_Export]):
 class CDFExporter(BaseExporter[T_VerifiedRules, T_Export]):
     @abstractmethod
     def export_to_cdf_iterable(
-        self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False
+        self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False, fallback_one_by_one: bool = False
     ) -> Iterable[UploadResult]:
         raise NotImplementedError
 
-    def export_to_cdf(self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False) -> UploadResultList:
-        return UploadResultList(self.export_to_cdf_iterable(rules, client, dry_run))
+    def export_to_cdf(
+        self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False, fallback_one_by_one: bool = False
+    ) -> UploadResultList:
+        return UploadResultList(self.export_to_cdf_iterable(rules, client, dry_run, fallback_one_by_one))

{cognite_neat-0.96.3 → cognite_neat-0.96.5}/cognite/neat/_rules/exporters/_rules2dms.py

@@ -118,12 +118,16 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
     def export(self, rules: DMSRules) -> DMSSchema:
         return rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
 
-    def delete_from_cdf(self, rules: DMSRules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
+    def delete_from_cdf(
+        self, rules: DMSRules, client: CogniteClient, dry_run: bool = False, skip_space: bool = False
+    ) -> Iterable[UploadResult]:
         to_export = self._prepare_exporters(rules, client)
 
         # we need to reverse order in which we are picking up the items to delete
         # as they are sorted in the order of creation and we need to delete them in reverse order
         for items, loader in reversed(to_export):
+            if skip_space and isinstance(loader, SpaceLoader):
+                continue
             item_ids = loader.get_ids(items)
             existing_items = loader.retrieve(item_ids)
             existing_ids = loader.get_ids(existing_items)
@@ -162,10 +166,15 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
             )
 
     def export_to_cdf_iterable(
-        self, rules: DMSRules, client: CogniteClient, dry_run: bool = False
+        self, rules: DMSRules, client: CogniteClient, dry_run: bool = False, fallback_one_by_one: bool = False
     ) -> Iterable[UploadResult]:
         to_export = self._prepare_exporters(rules, client)
 
+        result_by_name = {}
+        if self.existing_handling == "force":
+            for delete_result in self.delete_from_cdf(rules, client, dry_run, skip_space=True):
+                result_by_name[delete_result.name] = delete_result
+
         redeploy_data_model = False
         for items, loader in to_export:
             # The conversion from DMS to GraphQL does not seem to be triggered even if the views
@@ -183,8 +192,10 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
             created: set[Hashable] = set()
             skipped: set[Hashable] = set()
             changed: set[Hashable] = set()
+            deleted: set[Hashable] = set()
             failed_created: set[Hashable] = set()
             failed_changed: set[Hashable] = set()
+            failed_deleted: set[Hashable] = set()
             error_messages: list[str] = []
             if dry_run:
                 if self.existing_handling in ["update", "force"]:
@@ -200,7 +211,20 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
                     try:
                         loader.delete(to_delete)
                     except CogniteAPIError as e:
-                        error_messages.append(f"Failed delete: {e.message}")
+                        if fallback_one_by_one:
+                            for item in to_delete:
+                                try:
+                                    loader.delete([item])
+                                except CogniteAPIError as item_e:
+                                    failed_deleted.add(loader.get_id(item))
+                                    error_messages.append(f"Failed delete: {item_e!s}")
+                                else:
+                                    deleted.add(loader.get_id(item))
+                        else:
+                            error_messages.append(f"Failed delete: {e!s}")
+                            failed_deleted.update(loader.get_id(item) for item in e.failed + e.unknown)
+                    else:
+                        deleted.update(loader.get_id(item) for item in to_delete)
 
                 if isinstance(loader, DataModelingLoader):
                     to_create = loader.sort_by_dependencies(to_create)
@@ -208,9 +232,19 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
                 try:
                     loader.create(to_create)
                 except CogniteAPIError as e:
-                    failed_created.update(loader.get_id(item) for item in e.failed + e.unknown)
-                    created.update(loader.get_id(item) for item in e.successful)
-                    error_messages.append(e.message)
+                    if fallback_one_by_one:
+                        for item in to_create:
+                            try:
+                                loader.create([item])
+                            except CogniteAPIError as item_e:
+                                failed_created.add(loader.get_id(item))
+                                error_messages.append(f"Failed create: {item_e!s}")
+                            else:
+                                created.add(loader.get_id(item))
+                    else:
+                        failed_created.update(loader.get_id(item) for item in e.failed + e.unknown)
+                        created.update(loader.get_id(item) for item in e.successful)
+                        error_messages.append(f"Failed create: {e!s}")
                 else:
                     created.update(loader.get_id(item) for item in to_create)
 
@@ -218,9 +252,19 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
                     try:
                         loader.update(to_update)
                     except CogniteAPIError as e:
-                        failed_changed.update(loader.get_id(item) for item in e.failed + e.unknown)
-                        changed.update(loader.get_id(item) for item in e.successful)
-                        error_messages.append(e.message)
+                        if fallback_one_by_one:
+                            for item in to_update:
+                                try:
+                                    loader.update([item])
+                                except CogniteAPIError as e_item:
+                                    failed_changed.add(loader.get_id(item))
+                                    error_messages.append(f"Failed update: {e_item!s}")
+                                else:
+                                    changed.add(loader.get_id(item))
+                        else:
+                            failed_changed.update(loader.get_id(item) for item in e.failed + e.unknown)
+                            changed.update(loader.get_id(item) for item in e.successful)
+                            error_messages.append(f"Failed update: {e!s}")
                     else:
                         changed.update(loader.get_id(item) for item in to_update)
                 elif self.existing_handling == "skip":
@@ -228,14 +272,22 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
                 elif self.existing_handling == "fail":
                     failed_changed.update(loader.get_id(item) for item in to_update)
 
+            if loader.resource_name in result_by_name:
+                delete_result = result_by_name[loader.resource_name]
+                deleted.update(delete_result.deleted)
+                failed_deleted.update(delete_result.failed_deleted)
+                error_messages.extend(delete_result.error_messages)
+
             yield UploadResult(
                 name=loader.resource_name,
                 created=created,
                 changed=changed,
+                deleted=deleted,
                 unchanged={loader.get_id(item) for item in unchanged},
                 skipped=skipped,
                 failed_created=failed_created,
                 failed_changed=failed_changed,
+                failed_deleted=failed_deleted,
                 error_messages=error_messages,
                 issues=issue_list,
             )

{cognite_neat-0.96.3 → cognite_neat-0.96.5}/cognite/neat/_rules/models/dms/_rules.py

@@ -231,7 +231,9 @@ class DMSProperty(SheetRow):
     @field_serializer("connection", when_used="unless-none")
     def remove_defaults(self, value: Any, info: SerializationInfo) -> str:
         if isinstance(value, Entity) and (metadata := _metadata(info.context)):
-            default_type = f"{metadata.space}{self.view.external_id}.{self.view_property}"
+            default_type = f"{self.view.external_id}.{self.view_property}"
+            if isinstance(value, EdgeEntity) and value.edge_type and value.edge_type.space != metadata.space:
+                default_type = f"{metadata.space}{default_type}"
             return value.dump(space=metadata.space, version=metadata.version, type=default_type)
         return str(value)
 

{cognite_neat-0.96.3 → cognite_neat-0.96.5}/cognite/neat/_rules/models/entities/_single_value.py

@@ -187,10 +187,10 @@ class Entity(BaseModel, extra="ignore"):
         }
         if isinstance(defaults, dict):
             for key, value in defaults.items():
-                if key in model_dump and value == defaults.get(key):
+                if key in model_dump and model_dump[key] == value:
                     del model_dump[key]
-
-        args = ",".join(f"{k}={v}" for k, v in model_dump.items())
+        # Sorting to ensure deterministic order
+        args = ",".join(f"{k}={v}" for k, v in sorted(model_dump.items(), key=lambda x: x[0]))
         if self.prefix == Undefined or (isinstance(defaults, dict) and self.prefix == defaults.get("prefix")):
             base_id = str(self.suffix)
         else:

{cognite_neat-0.96.3 → cognite_neat-0.96.5}/cognite/neat/_session/_to.py

@@ -63,12 +63,18 @@ class CDFToAPI:
 
         return loader.load_into_cdf(self._client)
 
-    def data_model(self, existing_handling: Literal["fail", "skip", "update", "force"] = "skip", dry_run: bool = False):
+    def data_model(
+        self,
+        existing_handling: Literal["fail", "skip", "update", "force"] = "skip",
+        dry_run: bool = False,
+        fallback_one_by_one: bool = False,
+    ):
         """Export the verified DMS data model to CDF.
 
         Args:
             existing_handling: How to handle if component of data model exists. Defaults to "skip".
             dry_run: If True, no changes will be made to CDF. Defaults to False.
+            fallback_one_by_one: If True, will fall back to one-by-one upload if batch upload fails. Defaults to False.
 
         ... note::
 
@@ -88,7 +94,9 @@ class CDFToAPI:
 
         conversion_issues = IssueList(action="to.cdf.data_model")
         with catch_warnings(conversion_issues):
-            result = exporter.export_to_cdf(self._state.last_verified_dms_rules, self._client, dry_run)
+            result = exporter.export_to_cdf(
+                self._state.last_verified_dms_rules, self._client, dry_run, fallback_one_by_one
+            )
         result.insert(0, UploadResultCore(name="schema", issues=conversion_issues))
         self._state.outcome.append(result)
         print("You can inspect the details with the .inspect.outcome(...) method.")

{cognite_neat-0.96.3 → cognite_neat-0.96.5}/cognite/neat/_utils/cdf/loaders/_base.py

@@ -27,7 +27,7 @@ class ResourceLoader(
 
     @classmethod
     @abstractmethod
-    def get_id(cls, item: T_WriteClass | T_WritableCogniteResource) -> T_ID:
+    def get_id(cls, item: T_WriteClass | T_WritableCogniteResource | dict | T_ID) -> T_ID:
         raise NotImplementedError
 
     @classmethod

{cognite_neat-0.96.3 → cognite_neat-0.96.5}/cognite/neat/_utils/cdf/loaders/_data_modeling.py

@@ -80,8 +80,12 @@ class SpaceLoader(DataModelingLoader[str, SpaceApply, Space, SpaceApplyList, Spa
     resource_name = "spaces"
 
     @classmethod
-    def get_id(cls, item: Space | SpaceApply) -> str:
-        return item.space
+    def get_id(cls, item: Space | SpaceApply | str | dict) -> str:
+        if isinstance(item, Space | SpaceApply):
+            return item.space
+        if isinstance(item, dict):
+            return item["space"]
+        return item
 
     def create(self, items: Sequence[SpaceApply]) -> SpaceList:
         return self.client.data_modeling.spaces.apply(items)
@@ -149,8 +153,12 @@ class ViewLoader(DataModelingLoader[ViewId, ViewApply, View, ViewApplyList, View
         self._tried_force_deploy: set[ViewId] = set()
 
     @classmethod
-    def get_id(cls, item: View | ViewApply) -> ViewId:
-        return item.as_id()
+    def get_id(cls, item: View | ViewApply | ViewId | dict) -> ViewId:
+        if isinstance(item, View | ViewApply):
+            return item.as_id()
+        if isinstance(item, dict):
+            return ViewId.load(item)
+        return item
 
     def create(self, items: Sequence[ViewApply]) -> ViewList:
         if self.existing_handling == "force":
@@ -247,8 +255,12 @@ class ContainerLoader(DataModelingLoader[ContainerId, ContainerApply, Container,
         self._tried_force_deploy: set[ContainerId] = set()
 
     @classmethod
-    def get_id(cls, item: Container | ContainerApply) -> ContainerId:
-        return item.as_id()
+    def get_id(cls, item: Container | ContainerApply | ContainerId | dict) -> ContainerId:
+        if isinstance(item, Container | ContainerApply):
+            return item.as_id()
+        if isinstance(item, dict):
+            return ContainerId.load(item)
+        return item
 
     def sort_by_dependencies(self, items: Sequence[ContainerApply]) -> list[ContainerApply]:
         container_by_id = {container.as_id(): container for container in items}
@@ -292,8 +304,12 @@ class DataModelLoader(DataModelingLoader[DataModelId, DataModelApply, DataModel,
     resource_name = "data_models"
 
     @classmethod
-    def get_id(cls, item: DataModel | DataModelApply) -> DataModelId:
-        return item.as_id()
+    def get_id(cls, item: DataModel | DataModelApply | DataModelId | dict) -> DataModelId:
+        if isinstance(item, DataModel | DataModelApply):
+            return item.as_id()
+        if isinstance(item, dict):
+            return DataModelId.load(item)
+        return item
 
     def create(self, items: Sequence[DataModelApply]) -> DataModelList:
         return self.client.data_modeling.data_models.apply(items)

{cognite_neat-0.96.3 → cognite_neat-0.96.5}/cognite/neat/_utils/cdf/loaders/_ingestion.py

@@ -26,10 +26,16 @@ class TransformationLoader(
     resource_name = "transformations"
 
     @classmethod
-    def get_id(cls, item: Transformation | TransformationWrite) -> str:
-        if item.external_id is None:
-            raise ValueError(f"Transformation {item} does not have an external_id")
-        return item.external_id
+    def get_id(cls, item: Transformation | TransformationWrite | str | dict) -> str:
+        if isinstance(item, Transformation | TransformationWrite):
+            if item.external_id is None:
+                raise ValueError(f"Transformation {item} does not have an external_id")
+            return item.external_id
+        if isinstance(item, dict):
+            if item.get("externalId") is None:
+                raise ValueError(f"Transformation {item} does not have an external_id")
+            return item["externalId"]
+        return item
 
     def create(self, items: Sequence[TransformationWrite]) -> TransformationList:
         return self.client.transformations.create(items)
@@ -50,10 +56,16 @@ class RawDatabaseLoader(ResourceLoader[str, DatabaseWrite, Database, DatabaseWri
     resource_name = "databases"
 
     @classmethod
-    def get_id(cls, item: Database | DatabaseWrite) -> str:
-        if item.name is None:
-            raise ValueError(f"Database {item} does not have a name")
-        return item.name
+    def get_id(cls, item: Database | DatabaseWrite | str | dict) -> str:
+        if isinstance(item, Database | DatabaseWrite):
+            if item.name is None:
+                raise ValueError(f"Database {item} does not have a name")
+            return item.name
+        if isinstance(item, dict):
+            if item.get("name") is None:
+                raise ValueError(f"Database {item} does not have a name")
+            return item["name"]
+        return item
 
     def create(self, items: Sequence[DatabaseWrite]) -> DatabaseList:
         return self.client.raw.databases.create([item.name for item in items if item.name is not None])
@@ -78,8 +90,12 @@ class RawTableLoader(ResourceLoader[RawTableID, RawTableWrite, RawTable, RawTabl
     resource_name = "tables"
 
     @classmethod
-    def get_id(cls, item: RawTable | RawTableWrite) -> RawTableID:
-        return item.as_id()
+    def get_id(cls, item: RawTable | RawTableWrite | RawTableID | dict) -> RawTableID:
+        if isinstance(item, RawTable | RawTableWrite):
+            return item.as_id()
+        if isinstance(item, dict):
+            return RawTableID(database=item["database"], table=item["name"])
+        return item
 
     @overload
     def _groupby_database(self, items: Sequence[RawTableWrite]) -> Iterable[tuple[str, Iterable[RawTableWrite]]]: ...

cognite_neat-0.96.5/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.96.5"

{cognite_neat-0.96.3 → cognite_neat-0.96.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.96.3"
+version = "0.96.5"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.96.3/cognite/neat/_version.py

@@ -1 +0,0 @@
-__version__ = "0.96.3"