cognite-neat 0.96.4__tar.gz → 0.96.6__tar.gz

{cognite_neat-0.96.4 → cognite_neat-0.96.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.96.4
+Version: 0.96.6
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

{cognite_neat-0.96.4 → cognite_neat-0.96.6}/cognite/neat/_constants.py

@@ -36,6 +36,7 @@ COGNITE_MODELS = (
     DataModelId("cdf_cdm", "CogniteCore", "v1"),
     DataModelId("cdf_idm", "CogniteProcessIndustries", "v1"),
 )
+COGNITE_SPACES = frozenset(model.space for model in COGNITE_MODELS)
 DMS_LISTABLE_PROPERTY_LIMIT = 1000
 
 EXAMPLE_RULES = PACKAGE_DIRECTORY / "_rules" / "examples"

{cognite_neat-0.96.4 → cognite_neat-0.96.6}/cognite/neat/_issues/_base.py

@@ -109,7 +109,11 @@ class NeatIssue:
     def dump(self) -> dict[str, Any]:
         """Return a dictionary representation of the issue."""
         variables = vars(self)
-        output = {to_camel(key): self._dump_value(value) for key, value in variables.items() if value is not None}
+        output = {
+            to_camel(key): self._dump_value(value)
+            for key, value in variables.items()
+            if not (value is None or key.startswith("_"))
+        }
         output["NeatIssue"] = type(self).__name__
         return output
 

{cognite_neat-0.96.4 → cognite_neat-0.96.6}/cognite/neat/_rules/exporters/_base.py

@@ -32,9 +32,11 @@ class BaseExporter(ABC, Generic[T_VerifiedRules, T_Export]):
 class CDFExporter(BaseExporter[T_VerifiedRules, T_Export]):
     @abstractmethod
     def export_to_cdf_iterable(
-        self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False
+        self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False, fallback_one_by_one: bool = False
     ) -> Iterable[UploadResult]:
         raise NotImplementedError
 
-    def export_to_cdf(self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False) -> UploadResultList:
-        return UploadResultList(self.export_to_cdf_iterable(rules, client, dry_run))
+    def export_to_cdf(
+        self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False, fallback_one_by_one: bool = False
+    ) -> UploadResultList:
+        return UploadResultList(self.export_to_cdf_iterable(rules, client, dry_run, fallback_one_by_one))

{cognite_neat-0.96.4 → cognite_neat-0.96.6}/cognite/neat/_rules/exporters/_rules2dms.py

@@ -118,12 +118,16 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
     def export(self, rules: DMSRules) -> DMSSchema:
         return rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
 
-    def delete_from_cdf(self, rules: DMSRules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
+    def delete_from_cdf(
+        self, rules: DMSRules, client: CogniteClient, dry_run: bool = False, skip_space: bool = False
+    ) -> Iterable[UploadResult]:
         to_export = self._prepare_exporters(rules, client)
 
         # we need to reverse order in which we are picking up the items to delete
         # as they are sorted in the order of creation and we need to delete them in reverse order
         for items, loader in reversed(to_export):
+            if skip_space and isinstance(loader, SpaceLoader):
+                continue
             item_ids = loader.get_ids(items)
             existing_items = loader.retrieve(item_ids)
             existing_ids = loader.get_ids(existing_items)
@@ -162,10 +166,15 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
             )
 
     def export_to_cdf_iterable(
-        self, rules: DMSRules, client: CogniteClient, dry_run: bool = False
+        self, rules: DMSRules, client: CogniteClient, dry_run: bool = False, fallback_one_by_one: bool = False
     ) -> Iterable[UploadResult]:
         to_export = self._prepare_exporters(rules, client)
 
+        result_by_name = {}
+        if self.existing_handling == "force":
+            for delete_result in self.delete_from_cdf(rules, client, dry_run, skip_space=True):
+                result_by_name[delete_result.name] = delete_result
+
         redeploy_data_model = False
         for items, loader in to_export:
             # The conversion from DMS to GraphQL does not seem to be triggered even if the views
@@ -183,8 +192,10 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
             created: set[Hashable] = set()
             skipped: set[Hashable] = set()
             changed: set[Hashable] = set()
+            deleted: set[Hashable] = set()
             failed_created: set[Hashable] = set()
             failed_changed: set[Hashable] = set()
+            failed_deleted: set[Hashable] = set()
             error_messages: list[str] = []
             if dry_run:
                 if self.existing_handling in ["update", "force"]:
@@ -200,7 +211,20 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
                     try:
                         loader.delete(to_delete)
                     except CogniteAPIError as e:
-                        error_messages.append(f"Failed delete: {e.message}")
+                        if fallback_one_by_one:
+                            for item in to_delete:
+                                try:
+                                    loader.delete([item])
+                                except CogniteAPIError as item_e:
+                                    failed_deleted.add(loader.get_id(item))
+                                    error_messages.append(f"Failed delete: {item_e!s}")
+                                else:
+                                    deleted.add(loader.get_id(item))
+                        else:
+                            error_messages.append(f"Failed delete: {e!s}")
+                            failed_deleted.update(loader.get_id(item) for item in e.failed + e.unknown)
+                    else:
+                        deleted.update(loader.get_id(item) for item in to_delete)
 
                 if isinstance(loader, DataModelingLoader):
                     to_create = loader.sort_by_dependencies(to_create)
@@ -208,9 +232,19 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
                 try:
                     loader.create(to_create)
                 except CogniteAPIError as e:
-                    failed_created.update(loader.get_id(item) for item in e.failed + e.unknown)
-                    created.update(loader.get_id(item) for item in e.successful)
-                    error_messages.append(e.message)
+                    if fallback_one_by_one:
+                        for item in to_create:
+                            try:
+                                loader.create([item])
+                            except CogniteAPIError as item_e:
+                                failed_created.add(loader.get_id(item))
+                                error_messages.append(f"Failed create: {item_e!s}")
+                            else:
+                                created.add(loader.get_id(item))
+                    else:
+                        failed_created.update(loader.get_id(item) for item in e.failed + e.unknown)
+                        created.update(loader.get_id(item) for item in e.successful)
+                        error_messages.append(f"Failed create: {e!s}")
                 else:
                     created.update(loader.get_id(item) for item in to_create)
 
@@ -218,9 +252,19 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
                     try:
                         loader.update(to_update)
                     except CogniteAPIError as e:
-                        failed_changed.update(loader.get_id(item) for item in e.failed + e.unknown)
-                        changed.update(loader.get_id(item) for item in e.successful)
-                        error_messages.append(e.message)
+                        if fallback_one_by_one:
+                            for item in to_update:
+                                try:
+                                    loader.update([item])
+                                except CogniteAPIError as e_item:
+                                    failed_changed.add(loader.get_id(item))
+                                    error_messages.append(f"Failed update: {e_item!s}")
+                                else:
+                                    changed.add(loader.get_id(item))
+                        else:
+                            failed_changed.update(loader.get_id(item) for item in e.failed + e.unknown)
+                            changed.update(loader.get_id(item) for item in e.successful)
+                            error_messages.append(f"Failed update: {e!s}")
                     else:
                         changed.update(loader.get_id(item) for item in to_update)
                 elif self.existing_handling == "skip":
@@ -228,14 +272,22 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
                 elif self.existing_handling == "fail":
                     failed_changed.update(loader.get_id(item) for item in to_update)
 
+            if loader.resource_name in result_by_name:
+                delete_result = result_by_name[loader.resource_name]
+                deleted.update(delete_result.deleted)
+                failed_deleted.update(delete_result.failed_deleted)
+                error_messages.extend(delete_result.error_messages)
+
             yield UploadResult(
                 name=loader.resource_name,
                 created=created,
                 changed=changed,
+                deleted=deleted,
                 unchanged={loader.get_id(item) for item in unchanged},
                 skipped=skipped,
                 failed_created=failed_created,
                 failed_changed=failed_changed,
+                failed_deleted=failed_deleted,
                 error_messages=error_messages,
                 issues=issue_list,
             )

{cognite_neat-0.96.4 → cognite_neat-0.96.6}/cognite/neat/_rules/importers/_spreadsheet2rules.py

@@ -47,6 +47,8 @@ SOURCE_SHEET__TARGET_FIELD__HEADERS = [
     ("Classes", "Classes", "Class"),
     ("Containers", "Containers", "Container"),
     ("Views", "Views", "View"),
+    ("Enum", "Enum", "Collection"),
+    ("Nodes", "Nodes", "Node"),
 ]
 
 MANDATORY_SHEETS_BY_ROLE: dict[RoleTypes, set[str]] = {
@@ -112,7 +114,7 @@ class SpreadsheetReader:
         issue_list: IssueList,
         required: bool = True,
         metadata: MetadataRaw | None = None,
-        sheet_prefix: Literal["", "Last", "Ref"] = "",
+        sheet_prefix: Literal["", "Last", "Ref", "CDMRef"] = "",
     ):
         self.issue_list = issue_list
         self.required = required
@@ -267,6 +269,8 @@ class ExcelImporter(BaseImporter[T_InputRules]):
             reference_read: ReadResult | None = None
             if any(sheet_name.startswith("Ref") for sheet_name in user_reader.seen_sheets):
                 reference_read = SpreadsheetReader(issue_list, sheet_prefix="Ref").read(excel_file, self.filepath)
+            elif any(sheet_name.startswith("CDMRef") for sheet_name in user_reader.seen_sheets):
+                reference_read = SpreadsheetReader(issue_list, sheet_prefix="CDMRef").read(excel_file, self.filepath)
 
         if issue_list.has_errors:
             return ReadRules(None, issue_list, {})

{cognite_neat-0.96.4 → cognite_neat-0.96.6}/cognite/neat/_rules/models/dms/_exporter.py

@@ -402,17 +402,30 @@ class _DMSExporter:
 
         return container_properties_by_id, view_properties_by_id
 
-    @staticmethod
     def _gather_properties_with_ancestors(
+        self,
         view_properties_by_id: dict[dm.ViewId, list[DMSProperty]],
         views: Sequence[DMSView],
     ) -> dict[dm.ViewId, list[DMSProperty]]:
+        all_view_properties_by_id = view_properties_by_id.copy()
+        if self.rules.reference:
+            # We need to include t
+            ref_view_properties_by_id = self._gather_properties(self.rules.reference.properties)[1]
+            for view_id, properties in ref_view_properties_by_id.items():
+                if view_id not in all_view_properties_by_id:
+                    all_view_properties_by_id[view_id] = properties
+                else:
+                    existing_properties = {prop._identifier() for prop in all_view_properties_by_id[view_id]}
+                    for prop in properties:
+                        if prop._identifier() not in existing_properties:
+                            all_view_properties_by_id[view_id].append(prop)
+
         view_properties_with_parents_by_id: dict[dm.ViewId, list[DMSProperty]] = defaultdict(list)
         view_by_view_id = {view.view.as_id(): view for view in views}
         for view in views:
             view_id = view.view.as_id()
             seen: set[Hashable] = set()
-            if view_properties := view_properties_by_id.get(view_id):
+            if view_properties := all_view_properties_by_id.get(view_id):
                 view_properties_with_parents_by_id[view_id].extend(view_properties)
                 seen.update(prop._identifier() for prop in view_properties)
             if not view.implements:
@@ -428,7 +441,7 @@ class _DMSExporter:
                             parents.append(grandparent)
                             seen_parents.add(grandparent)
 
-                if not (parent_view_properties := view_properties_by_id.get(parent_view_id)):
+                if not (parent_view_properties := all_view_properties_by_id.get(parent_view_id)):
                     continue
                 for prop in parent_view_properties:
                     new_prop = prop.model_copy(update={"view": view.view})

{cognite_neat-0.96.4 → cognite_neat-0.96.6}/cognite/neat/_rules/models/dms/_rules.py

@@ -1,15 +1,15 @@
 import math
-import sys
 import warnings
 from collections.abc import Hashable
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, ClassVar, Literal
+from typing import Any, ClassVar, Literal
 
 import pandas as pd
 from cognite.client import data_modeling as dm
 from pydantic import Field, field_serializer, field_validator, model_validator
 from pydantic_core.core_schema import SerializationInfo, ValidationInfo
 
+from cognite.neat._constants import COGNITE_SPACES
 from cognite.neat._issues import MultiValueError
 from cognite.neat._issues.warnings import (
     PrincipleMatchingSpaceAndVersionWarning,
@@ -56,14 +56,6 @@ from cognite.neat._rules.models.entities import (
 
 from ._schema import DMSSchema
 
-if TYPE_CHECKING:
-    pass
-
-if sys.version_info >= (3, 11):
-    pass
-else:
-    pass
-
 _DEFAULT_VERSION = "1"
 
 
@@ -197,6 +189,24 @@ class DMSProperty(SheetRow):
             raise ValueError(f"Reverse connection must have a value type that points to a view, got {value}")
         return value
 
+    @field_validator("container", "container_property", mode="after")
+    def container_set_correctly(cls, value: Any, info: ValidationInfo) -> Any:
+        if (connection := info.data.get("connection")) is None:
+            return value
+        if connection == "direct" and value is None:
+            raise ValueError(
+                "You must provide a container and container property for where to store direct connections"
+            )
+        elif isinstance(connection, EdgeEntity) and value is not None:
+            raise ValueError(
+                "Edge connections are not stored in a container, please remove the container and container property"
+            )
+        elif isinstance(connection, ReverseConnectionEntity) and value is not None:
+            raise ValueError(
+                "Reverse connection are not stored in a container, please remove the container and container property"
+            )
+        return value
+
     @field_serializer("reference", when_used="always")
     def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
         if isinstance(info.context, dict) and info.context.get("as_reference") is True:
@@ -421,7 +431,11 @@ class DMSRules(BaseRules):
         if not (metadata := info.data.get("metadata")):
             return value
         model_version = metadata.version
-        if different_version := [view.view.as_id() for view in value if view.view.version != model_version]:
+        if different_version := [
+            view.view.as_id()
+            for view in value
+            if view.view.version != model_version and view.view.space not in COGNITE_SPACES
+        ]:
             for view_id in different_version:
                 warnings.warn(
                     PrincipleMatchingSpaceAndVersionWarning(
@@ -429,7 +443,11 @@ class DMSRules(BaseRules):
                     ),
                     stacklevel=2,
                 )
-        if different_space := [view.view.as_id() for view in value if view.view.space != metadata.space]:
+        if different_space := [
+            view.view.as_id()
+            for view in value
+            if view.view.space != metadata.space and view.view.space not in COGNITE_SPACES
+        ]:
             for view_id in different_space:
                 warnings.warn(
                     PrincipleMatchingSpaceAndVersionWarning(

{cognite_neat-0.96.4 → cognite_neat-0.96.6}/cognite/neat/_session/_to.py

@@ -63,12 +63,18 @@ class CDFToAPI:
 
         return loader.load_into_cdf(self._client)
 
-    def data_model(self, existing_handling: Literal["fail", "skip", "update", "force"] = "skip", dry_run: bool = False):
+    def data_model(
+        self,
+        existing_handling: Literal["fail", "skip", "update", "force"] = "skip",
+        dry_run: bool = False,
+        fallback_one_by_one: bool = False,
+    ):
         """Export the verified DMS data model to CDF.
 
         Args:
             existing_handling: How to handle if component of data model exists. Defaults to "skip".
             dry_run: If True, no changes will be made to CDF. Defaults to False.
+            fallback_one_by_one: If True, will fall back to one-by-one upload if batch upload fails. Defaults to False.
 
         ... note::
 
@@ -88,7 +94,9 @@ class CDFToAPI:
 
         conversion_issues = IssueList(action="to.cdf.data_model")
         with catch_warnings(conversion_issues):
-            result = exporter.export_to_cdf(self._state.last_verified_dms_rules, self._client, dry_run)
+            result = exporter.export_to_cdf(
+                self._state.last_verified_dms_rules, self._client, dry_run, fallback_one_by_one
+            )
         result.insert(0, UploadResultCore(name="schema", issues=conversion_issues))
         self._state.outcome.append(result)
         print("You can inspect the details with the .inspect.outcome(...) method.")

{cognite_neat-0.96.4 → cognite_neat-0.96.6}/cognite/neat/_utils/cdf/loaders/_base.py

@@ -27,7 +27,7 @@ class ResourceLoader(
 
     @classmethod
     @abstractmethod
-    def get_id(cls, item: T_WriteClass | T_WritableCogniteResource) -> T_ID:
+    def get_id(cls, item: T_WriteClass | T_WritableCogniteResource | dict | T_ID) -> T_ID:
         raise NotImplementedError
 
     @classmethod

{cognite_neat-0.96.4 → cognite_neat-0.96.6}/cognite/neat/_utils/cdf/loaders/_data_modeling.py

@@ -80,8 +80,12 @@ class SpaceLoader(DataModelingLoader[str, SpaceApply, Space, SpaceApplyList, Spa
     resource_name = "spaces"
 
     @classmethod
-    def get_id(cls, item: Space | SpaceApply) -> str:
-        return item.space
+    def get_id(cls, item: Space | SpaceApply | str | dict) -> str:
+        if isinstance(item, Space | SpaceApply):
+            return item.space
+        if isinstance(item, dict):
+            return item["space"]
+        return item
 
     def create(self, items: Sequence[SpaceApply]) -> SpaceList:
         return self.client.data_modeling.spaces.apply(items)
@@ -149,8 +153,12 @@ class ViewLoader(DataModelingLoader[ViewId, ViewApply, View, ViewApplyList, View
         self._tried_force_deploy: set[ViewId] = set()
 
     @classmethod
-    def get_id(cls, item: View | ViewApply) -> ViewId:
-        return item.as_id()
+    def get_id(cls, item: View | ViewApply | ViewId | dict) -> ViewId:
+        if isinstance(item, View | ViewApply):
+            return item.as_id()
+        if isinstance(item, dict):
+            return ViewId.load(item)
+        return item
 
     def create(self, items: Sequence[ViewApply]) -> ViewList:
         if self.existing_handling == "force":
@@ -247,8 +255,12 @@ class ContainerLoader(DataModelingLoader[ContainerId, ContainerApply, Container,
         self._tried_force_deploy: set[ContainerId] = set()
 
     @classmethod
-    def get_id(cls, item: Container | ContainerApply) -> ContainerId:
-        return item.as_id()
+    def get_id(cls, item: Container | ContainerApply | ContainerId | dict) -> ContainerId:
+        if isinstance(item, Container | ContainerApply):
+            return item.as_id()
+        if isinstance(item, dict):
+            return ContainerId.load(item)
+        return item
 
     def sort_by_dependencies(self, items: Sequence[ContainerApply]) -> list[ContainerApply]:
         container_by_id = {container.as_id(): container for container in items}
@@ -292,8 +304,12 @@ class DataModelLoader(DataModelingLoader[DataModelId, DataModelApply, DataModel,
     resource_name = "data_models"
 
     @classmethod
-    def get_id(cls, item: DataModel | DataModelApply) -> DataModelId:
-        return item.as_id()
+    def get_id(cls, item: DataModel | DataModelApply | DataModelId | dict) -> DataModelId:
+        if isinstance(item, DataModel | DataModelApply):
+            return item.as_id()
+        if isinstance(item, dict):
+            return DataModelId.load(item)
+        return item
 
     def create(self, items: Sequence[DataModelApply]) -> DataModelList:
         return self.client.data_modeling.data_models.apply(items)

{cognite_neat-0.96.4 → cognite_neat-0.96.6}/cognite/neat/_utils/cdf/loaders/_ingestion.py

@@ -26,10 +26,16 @@ class TransformationLoader(
     resource_name = "transformations"
 
     @classmethod
-    def get_id(cls, item: Transformation | TransformationWrite) -> str:
-        if item.external_id is None:
-            raise ValueError(f"Transformation {item} does not have an external_id")
-        return item.external_id
+    def get_id(cls, item: Transformation | TransformationWrite | str | dict) -> str:
+        if isinstance(item, Transformation | TransformationWrite):
+            if item.external_id is None:
+                raise ValueError(f"Transformation {item} does not have an external_id")
+            return item.external_id
+        if isinstance(item, dict):
+            if item.get("externalId") is None:
+                raise ValueError(f"Transformation {item} does not have an external_id")
+            return item["externalId"]
+        return item
 
     def create(self, items: Sequence[TransformationWrite]) -> TransformationList:
         return self.client.transformations.create(items)
@@ -50,10 +56,16 @@ class RawDatabaseLoader(ResourceLoader[str, DatabaseWrite, Database, DatabaseWri
     resource_name = "databases"
 
     @classmethod
-    def get_id(cls, item: Database | DatabaseWrite) -> str:
-        if item.name is None:
-            raise ValueError(f"Database {item} does not have a name")
-        return item.name
+    def get_id(cls, item: Database | DatabaseWrite | str | dict) -> str:
+        if isinstance(item, Database | DatabaseWrite):
+            if item.name is None:
+                raise ValueError(f"Database {item} does not have a name")
+            return item.name
+        if isinstance(item, dict):
+            if item.get("name") is None:
+                raise ValueError(f"Database {item} does not have a name")
+            return item["name"]
+        return item
 
     def create(self, items: Sequence[DatabaseWrite]) -> DatabaseList:
         return self.client.raw.databases.create([item.name for item in items if item.name is not None])
@@ -78,8 +90,12 @@ class RawTableLoader(ResourceLoader[RawTableID, RawTableWrite, RawTable, RawTabl
     resource_name = "tables"
 
     @classmethod
-    def get_id(cls, item: RawTable | RawTableWrite) -> RawTableID:
-        return item.as_id()
+    def get_id(cls, item: RawTable | RawTableWrite | RawTableID | dict) -> RawTableID:
+        if isinstance(item, RawTable | RawTableWrite):
+            return item.as_id()
+        if isinstance(item, dict):
+            return RawTableID(database=item["database"], table=item["name"])
+        return item
 
     @overload
     def _groupby_database(self, items: Sequence[RawTableWrite]) -> Iterable[tuple[str, Iterable[RawTableWrite]]]: ...

cognite_neat-0.96.6/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.96.6"

{cognite_neat-0.96.4 → cognite_neat-0.96.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.96.4"
+version = "0.96.6"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.96.4/cognite/neat/_version.py

@@ -1 +0,0 @@
-__version__ = "0.96.4"