cognite-neat 0.82.3__py3-none-any.whl → 0.84.0__py3-none-any.whl

cognite/neat/_shared.py

@@ -0,0 +1,35 @@
+from abc import abstractmethod
+from collections.abc import Hashable, Sequence
+from dataclasses import dataclass
+from typing import Any, TypeVar
+
+import pandas as pd
+
+T_ID = TypeVar("T_ID", bound=Hashable)
+
+
+@dataclass
+class NeatObject:
+    """A neat object can be dumped to a dictionary."""
+
+    @abstractmethod
+    def dump(self, aggregate: bool = True) -> dict[str, Any]:
+        """Return a dictionary representation of the object."""
+        raise NotImplementedError()
+
+
+T_NeatObject = TypeVar("T_NeatObject", bound=NeatObject)
+
+
+class NeatList(list, Sequence[T_NeatObject]):
+    """A list of neat objects."""
+
+    def dump(self) -> list[dict[str, Any]]:
+        """Return a list of dictionary representations of the objects."""
+        return [obj.dump() for obj in self]
+
+    def to_pandas(self) -> pd.DataFrame:
+        return pd.DataFrame(self.dump())
+
+    def _repr_html_(self) -> str:
+        return self.to_pandas()._repr_html_()  # type: ignore[operator]

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.82.3"
+__version__ = "0.84.0"

cognite/neat/legacy/rules/examples/power-grid-containers.yaml

@@ -3,6 +3,7 @@
   properties:
     name:
       autoIncrement: false
+      immutable: false
       defaultValue: null
       description: 'The name that identifies Greographical
 
@@ -20,6 +21,7 @@
   properties:
     aliasName:
       autoIncrement: false
+      immutable: false
       defaultValue: null
       description: 'The alternative name that identifies Substation
 
@@ -32,6 +34,7 @@
         type: text
     name:
       autoIncrement: false
+      immutable: false
       defaultValue: null
       description: 'The name that identifies Terminal
 
@@ -44,6 +47,7 @@
         type: text
     substation:
       autoIncrement: false
+      immutable: false
       defaultValue: null
       description: 'Substation to which terminal belongs to
 
@@ -61,6 +65,7 @@
   properties:
     name:
       autoIncrement: false
+      immutable: false
       defaultValue: null
       description: 'The name that identifies SubGreographical
 
@@ -73,6 +78,7 @@
         type: text
     region:
       autoIncrement: false
+      immutable: false
       defaultValue: null
       description: 'Region to which subgeographical region belongs to
 
@@ -90,6 +96,7 @@
   properties:
     name:
       autoIncrement: false
+      immutable: false
       defaultValue: null
       description: 'The name that identifies Substation
 
@@ -102,6 +109,7 @@
         type: text
     subGeographicalRegion:
       autoIncrement: false
+      immutable: false
       defaultValue: null
       description: 'The subgeographical region containing the substation
 

cognite/neat/legacy/rules/examples/power-grid-model.yaml

@@ -23,6 +23,7 @@ views:
           type: text
         nullable: false
         autoIncrement: false
+        immutable: false
         source: null
         defaultValue: null
         name: name
@@ -67,6 +68,7 @@ views:
           type: text
         nullable: false
         autoIncrement: false
+        immutable: false
         source: null
         defaultValue: null
         name: name
@@ -88,6 +90,7 @@ views:
             version: "1"
         nullable: true
         autoIncrement: false
+        immutable: false
         defaultValue: null
         name: region
         description: "Region to which subgeographical region belongs to
@@ -119,6 +122,7 @@ views:
           type: text
         nullable: false
         autoIncrement: false
+        immutable: false
         source: null
         defaultValue: null
         name: name
@@ -140,6 +144,7 @@ views:
             version: bbe4295878a47f
         nullable: true
         autoIncrement: false
+        immutable: false
         defaultValue: null
         name: subGeographicalRegion
         description: "The subgeographical region containing the substation
@@ -166,6 +171,7 @@ views:
           type: text
         nullable: false
         autoIncrement: false
+        immutable: false
         source: null
         defaultValue: null
         name: name
@@ -183,6 +189,7 @@ views:
           type: text
         nullable: true
         autoIncrement: false
+        immutable: false
         source: null
         defaultValue: null
         name: aliasName
@@ -204,6 +211,7 @@ views:
             version: "1"
         nullable: true
         autoIncrement: false
+        immutable: false
         defaultValue: null
         name: substation
         description: "Substation to which terminal belongs to

cognite/neat/rules/exporters/_base.py

@@ -7,7 +7,7 @@ from cognite.client import CogniteClient
 
 from cognite.neat.rules._shared import Rules
 from cognite.neat.rules.models import DMSRules, InformationRules, RoleTypes
-from cognite.neat.utils.upload import UploadDiffsCount
+from cognite.neat.utils.upload import UploadResult, UploadResultList
 
 T_Export = TypeVar("T_Export")
 
@@ -37,5 +37,10 @@ class BaseExporter(ABC, Generic[T_Export]):
 
 class CDFExporter(BaseExporter[T_Export]):
     @abstractmethod
-    def export_to_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadDiffsCount]:
+    def export_to_cdf_iterable(
+        self, rules: Rules, client: CogniteClient, dry_run: bool = False
+    ) -> Iterable[UploadResult]:
         raise NotImplementedError
+
+    def export_to_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> UploadResultList:
+        return UploadResultList(self.export_to_cdf_iterable(rules, client, dry_run))

cognite/neat/rules/exporters/_rules2dms.py

@@ -1,5 +1,5 @@
 import warnings
-from collections.abc import Collection, Iterable
+from collections.abc import Collection, Hashable, Iterable, Sequence
 from pathlib import Path
 from typing import Literal, TypeAlias, cast
 
@@ -31,7 +31,7 @@ from cognite.neat.utils.cdf_loaders import (
     TransformationLoader,
     ViewLoader,
 )
-from cognite.neat.utils.upload import UploadDiffsCount
+from cognite.neat.utils.upload import UploadResult
 
 from ._base import CDFExporter
 
@@ -123,109 +123,84 @@ class DMSExporter(CDFExporter[DMSSchema]):
             raise ValueError(f"{type(rules).__name__} cannot be exported to DMS")
         return dms_rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
 
-    def delete_from_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadDiffsCount]:
-        schema, to_export = self._prepare_schema_and_exporters(rules, client)
+    def delete_from_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
+        to_export = self._prepare_exporters(rules, client)
 
         # we need to reverse order in which we are picking up the items to delete
         # as they are sorted in the order of creation and we need to delete them in reverse order
-        for all_items, loader in reversed(to_export):
-            all_item_ids = loader.get_ids(all_items)
-            item_ids = [item_id for item_id in all_item_ids]
-            cdf_items = loader.retrieve(item_ids)
-            cdf_item_by_id = {loader.get_id(item): item for item in cdf_items}
-            items = [item for item in all_items if loader.get_id(item) in item_ids]
-            to_delete = []
-
-            for item in items:
+        for items, loader in reversed(to_export):
+            item_ids = loader.get_ids(items)
+            existing_items = loader.retrieve(item_ids)
+            existing_ids = loader.get_ids(existing_items)
+            to_delete: list[Hashable] = []
+            for item_id in item_ids:
                 if (
                     isinstance(loader, DataModelingLoader)
                     and self.include_space is not None
-                    and not loader.in_space(item, self.include_space)
+                    and not loader.in_space(item_id, self.include_space)
                 ):
                     continue
 
-                cdf_item = cdf_item_by_id.get(loader.get_id(item))
-                if cdf_item:
-                    to_delete.append(cdf_item)
-
-            deleted = len(to_delete)
-            failed_deleted = 0
+                if item_id in existing_ids:
+                    to_delete.append(item_id)
 
+            deleted: set[Hashable] = set()
+            failed_deleted: set[Hashable] = set()
             error_messages: list[str] = []
-            if not dry_run:
-                if to_delete:
-                    try:
-                        loader.delete(to_delete)
-                    except CogniteAPIError as e:
-                        failed_deleted = len(e.failed) + len(e.unknown)
-                        deleted -= failed_deleted
-                        error_messages.append(f"Failed delete: {e.message}")
+            if dry_run:
+                deleted.update(to_delete)
+            elif to_delete:
+                try:
+                    loader.delete(to_delete)
+                except CogniteAPIError as e:
+                    failed_deleted.update(loader.get_id(item) for item in e.failed + e.unknown)
+                    deleted.update(loader.get_id(item) for item in e.successful)
+                    error_messages.append(f"Failed delete: {e.message}")
+                else:
+                    deleted.update(to_delete)
 
-            yield UploadDiffsCount(
+            yield UploadResult(
                 name=loader.resource_name,
                 deleted=deleted,
-                skipped=0,
                 failed_deleted=failed_deleted,
                 error_messages=error_messages,
             )
 
-    def export_to_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadDiffsCount]:
-        schema, to_export = self._prepare_schema_and_exporters(rules, client)
+    def export_to_cdf_iterable(
+        self, rules: Rules, client: CogniteClient, dry_run: bool = False
+    ) -> Iterable[UploadResult]:
+        to_export = self._prepare_exporters(rules, client)
 
-        # The conversion from DMS to GraphQL does not seem to be triggered even if the views
-        # are changed. This is a workaround to force the conversion.
         redeploy_data_model = False
+        for items, loader in to_export:
+            # The conversion from DMS to GraphQL does not seem to be triggered even if the views
+            # are changed. This is a workaround to force the conversion.
+            is_redeploying = loader is DataModelingLoader and redeploy_data_model
 
-        for all_items, loader in to_export:
-            issue_list = IssueList()
-            all_item_ids = loader.get_ids(all_items)
-            item_ids = [item_id for item_id in all_item_ids]
-            cdf_items = loader.retrieve(item_ids)
-            cdf_item_by_id = {loader.get_id(item): item for item in cdf_items}
-            items = [item for item in all_items if loader.get_id(item) in item_ids]
-            to_create, to_update, unchanged, to_delete = [], [], [], []
-            is_redeploying = loader.resource_name == "data_models" and redeploy_data_model
-            for item in items:
-                if (
-                    isinstance(loader, DataModelingLoader)
-                    and self.include_space is not None
-                    and not loader.in_space(item, self.include_space)
-                ):
-                    continue
-
-                cdf_item = cdf_item_by_id.get(loader.get_id(item))
-                if cdf_item is None:
-                    to_create.append(item)
-                elif is_redeploying:
-                    to_update.append(item)
-                    to_delete.append(cdf_item)
-                elif loader.are_equal(item, cdf_item):
-                    unchanged.append(item)
-                else:
-                    to_update.append(item)
-
-            created = len(to_create)
-            failed_created = 0
-            skipped = 0
-
-            if self.existing_handling in ["update", "force"]:
-                changed = len(to_update)
-                failed_changed = 0
-            elif self.existing_handling == "skip":
-                changed = 0
-                failed_changed = 0
-                skipped += len(to_update)
-            elif self.existing_handling == "fail":
-                failed_changed = len(to_update)
-                changed = 0
-            else:
-                raise ValueError(f"Unsupported existing_handling {self.existing_handling}")
+            to_create, to_delete, to_update, unchanged = self._categorize_items_for_upload(
+                loader, items, is_redeploying
+            )
 
+            issue_list = IssueList()
             warning_list = self._validate(loader, items)
             issue_list.extend(warning_list)
 
+            created: set[Hashable] = set()
+            skipped: set[Hashable] = set()
+            changed: set[Hashable] = set()
+            failed_created: set[Hashable] = set()
+            failed_changed: set[Hashable] = set()
             error_messages: list[str] = []
-            if not dry_run:
+            if dry_run:
+                if self.existing_handling in ["update", "force"]:
+                    changed.update(loader.get_id(item) for item in to_update)
+                elif self.existing_handling == "skip":
+                    skipped.update(loader.get_id(item) for item in to_update)
+                elif self.existing_handling == "fail":
+                    failed_changed.update(loader.get_id(item) for item in to_update)
+                else:
+                    raise ValueError(f"Unsupported existing_handling {self.existing_handling}")
+            else:
                 if to_delete:
                     try:
                         loader.delete(to_delete)
@@ -238,23 +213,31 @@ class DMSExporter(CDFExporter[DMSSchema]):
                 try:
                     loader.create(to_create)
                 except CogniteAPIError as e:
-                    failed_created = len(e.failed) + len(e.unknown)
-                    created -= failed_created
+                    failed_created.update(loader.get_id(item) for item in e.failed + e.unknown)
+                    created.update(loader.get_id(item) for item in e.successful)
                     error_messages.append(e.message)
+                else:
+                    created.update(loader.get_id(item) for item in to_create)
 
                 if self.existing_handling in ["update", "force"]:
                     try:
                         loader.update(to_update)
                     except CogniteAPIError as e:
-                        failed_changed = len(e.failed) + len(e.unknown)
-                        changed -= failed_changed
+                        failed_changed.update(loader.get_id(item) for item in e.failed + e.unknown)
+                        changed.update(loader.get_id(item) for item in e.successful)
                         error_messages.append(e.message)
-
-            yield UploadDiffsCount(
+                    else:
+                        changed.update(loader.get_id(item) for item in to_update)
+                elif self.existing_handling == "skip":
+                    skipped.update(loader.get_id(item) for item in to_update)
+                elif self.existing_handling == "fail":
+                    failed_changed.update(loader.get_id(item) for item in to_update)
+
+            yield UploadResult(
                 name=loader.resource_name,
                 created=created,
                 changed=changed,
-                unchanged=len(unchanged),
+                unchanged={loader.get_id(item) for item in unchanged},
                 skipped=skipped,
                 failed_created=failed_created,
                 failed_changed=failed_changed,
@@ -262,12 +245,37 @@ class DMSExporter(CDFExporter[DMSSchema]):
                 issues=issue_list,
             )
 
-            if loader.resource_name == "views" and (created or changed) and not redeploy_data_model:
+            if loader is ViewLoader and (created or changed):
                 redeploy_data_model = True
 
-    def _prepare_schema_and_exporters(
-        self, rules, client
-    ) -> tuple[DMSSchema, list[tuple[CogniteResourceList, ResourceLoader]]]:
+    def _categorize_items_for_upload(
+        self, loader: ResourceLoader, items: Sequence[CogniteResource], is_redeploying
+    ) -> tuple[list[CogniteResource], list[CogniteResource], list[CogniteResource], list[CogniteResource]]:
+        item_ids = loader.get_ids(items)
+        cdf_items = loader.retrieve(item_ids)
+        cdf_item_by_id = {loader.get_id(item): item for item in cdf_items}
+        to_create, to_update, unchanged, to_delete = [], [], [], []
+        for item in items:
+            if (
+                isinstance(loader, DataModelingLoader)
+                and self.include_space is not None
+                and not loader.in_space(item, self.include_space)
+            ):
+                continue
+
+            cdf_item = cdf_item_by_id.get(loader.get_id(item))
+            if cdf_item is None:
+                to_create.append(item)
+            elif is_redeploying:
+                to_update.append(item)
+                to_delete.append(cdf_item)
+            elif loader.are_equal(item, cdf_item):
+                unchanged.append(item)
+            else:
+                to_update.append(item)
+        return to_create, to_delete, to_update, unchanged
+
+    def _prepare_exporters(self, rules, client) -> list[tuple[CogniteResourceList, ResourceLoader]]:
         schema = self.export(rules)
         to_export: list[tuple[CogniteResourceList, ResourceLoader]] = []
         if self.export_components.intersection({"all", "spaces"}):
@@ -282,9 +290,9 @@ class DMSExporter(CDFExporter[DMSSchema]):
             to_export.append((schema.databases, RawDatabaseLoader(client)))
             to_export.append((schema.raw_tables, RawTableLoader(client)))
             to_export.append((schema.transformations, TransformationLoader(client)))
-        return schema, to_export
+        return to_export
 
-    def _validate(self, loader: ResourceLoader, items: list[CogniteResource]) -> IssueList:
+    def _validate(self, loader: ResourceLoader, items: CogniteResourceList) -> IssueList:
         issue_list = IssueList()
         if isinstance(loader, DataModelLoader):
             models = cast(list[DataModelApply], items)

cognite/neat/rules/models/dms/_schema.py

@@ -746,6 +746,7 @@ class DMSSchema:
                 type=container_prop.type,
                 nullable=container_prop.nullable,
                 auto_increment=container_prop.auto_increment,
+                immutable=container_prop.immutable,
                 # Likely bug in SDK.
                 default_value=container_prop.default_value,  # type: ignore[arg-type]
             )

cognite/neat/utils/cdf_loaders/_base.py

@@ -9,18 +9,10 @@ from cognite.client.data_classes._base import (
     T_WriteClass,
     WriteableCogniteResourceList,
 )
-from cognite.client.data_classes.data_modeling import (
-    DataModelingId,
-)
-from cognite.client.data_classes.data_modeling.ids import (
-    InstanceId,
-    VersionedDataModelingId,
-)
 from cognite.client.utils.useful_types import SequenceNotStr
 
-from .data_classes import RawTableID
+from cognite.neat._shared import T_ID
 
-T_ID = TypeVar("T_ID", bound=str | int | DataModelingId | InstanceId | VersionedDataModelingId | RawTableID)
 T_WritableCogniteResourceList = TypeVar("T_WritableCogniteResourceList", bound=WriteableCogniteResourceList)
 
 

cognite/neat/utils/cdf_loaders/_data_modeling.py

@@ -41,7 +41,8 @@ from ._base import T_ID, ResourceLoader, T_WritableCogniteResourceList
 class DataModelingLoader(
     ResourceLoader[T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList]
 ):
-    def in_space(self, item: T_WriteClass | T_WritableCogniteResource, space: set[str]) -> bool:
+    @classmethod
+    def in_space(cls, item: T_WriteClass | T_WritableCogniteResource | T_ID, space: set[str]) -> bool:
         if hasattr(item, "space"):
             return item.space in space
         raise ValueError(f"Item {item} does not have a space attribute")

cognite/neat/utils/upload.py

@@ -1,69 +1,89 @@
 from abc import ABC
 from dataclasses import dataclass, field
 from functools import total_ordering
+from typing import Any, Generic
 
+from cognite.neat._shared import T_ID, NeatList, NeatObject
 from cognite.neat.issues import NeatIssueList
 
 
 @total_ordering
 @dataclass
-class UploadResultCore(ABC):
+class UploadResultCore(NeatObject, ABC):
     name: str
     error_messages: list[str] = field(default_factory=list)
     issues: NeatIssueList = field(default_factory=NeatIssueList)
 
     def __lt__(self, other: object) -> bool:
-        if isinstance(other, UploadDiffsCount):
+        if isinstance(other, UploadResultCore):
             return self.name < other.name
         else:
             return NotImplemented
 
     def __eq__(self, other: object) -> bool:
-        if isinstance(other, UploadDiffsCount):
+        if isinstance(other, UploadResultCore):
             return self.name == other.name
         else:
             return NotImplemented
 
+    def dump(self, aggregate: bool = True) -> dict[str, Any]:
+        return {"name": self.name}
+
+
+class UploadResultList(NeatList[UploadResultCore]): ...
+
 
 @dataclass
-class UploadDiffsCount(UploadResultCore):
-    created: int = 0
-    deleted: int = 0
-    changed: int = 0
-    unchanged: int = 0
-    skipped: int = 0
-    failed_created: int = 0
-    failed_changed: int = 0
-    failed_deleted: int = 0
+class UploadResult(UploadResultCore, Generic[T_ID]):
+    created: set[T_ID] = field(default_factory=set)
+    deleted: set[T_ID] = field(default_factory=set)
+    changed: set[T_ID] = field(default_factory=set)
+    unchanged: set[T_ID] = field(default_factory=set)
+    skipped: set[T_ID] = field(default_factory=set)
+    failed_created: set[T_ID] = field(default_factory=set)
+    failed_changed: set[T_ID] = field(default_factory=set)
+    failed_deleted: set[T_ID] = field(default_factory=set)
 
     @property
-    def total(self) -> int:
-        return self.created + self.deleted + self.changed + self.unchanged
+    def failed(self) -> int:
+        return len(self.failed_created) + len(self.failed_changed) + len(self.failed_deleted)
 
     @property
-    def failed(self) -> int:
-        return self.failed_created + self.failed_changed + self.failed_deleted
+    def total(self) -> int:
+        return len(self.created) + len(self.deleted) + len(self.changed) + len(self.unchanged) + len(self.skipped)
 
-    def as_report_str(self) -> str:
-        line = []
+    def dump(self, aggregate: bool = True) -> dict[str, Any]:
+        output = super().dump(aggregate)
         if self.created:
-            line.append(f"created {self.created}")
+            output["created"] = len(self.created) if aggregate else list(self.created)
+        if self.deleted:
+            output["deleted"] = len(self.deleted) if aggregate else list(self.deleted)
         if self.changed:
-            line.append(f"updated {self.changed}")
-        if self.skipped:
-            line.append(f"skipped {self.skipped}")
+            output["changed"] = len(self.changed) if aggregate else list(self.changed)
         if self.unchanged:
-            line.append(f"unchanged {self.unchanged}")
-        if self.deleted:
-            line.append(f"deleted {self.deleted}")
+            output["unchanged"] = len(self.unchanged) if aggregate else list(self.unchanged)
+        if self.skipped:
+            output["skipped"] = len(self.skipped) if aggregate else list(self.skipped)
         if self.failed_created:
-            line.append(f"failed to create {self.failed_created}")
+            output["failed_created"] = len(self.failed_created) if aggregate else list(self.failed_created)
         if self.failed_changed:
-            line.append(f"failed to update {self.failed_changed}")
+            output["failed_changed"] = len(self.failed_changed) if aggregate else list(self.failed_changed)
         if self.failed_deleted:
-            line.append(f"failed to delete {self.failed_deleted}")
-
-        return f"{self.name.title()}: {', '.join(line)}"
+            output["failed_deleted"] = len(self.failed_deleted) if aggregate else list(self.failed_deleted)
+        if self.error_messages:
+            output["error_messages"] = len(self.error_messages) if aggregate else self.error_messages
+        if self.issues:
+            output["issues"] = len(self.issues) if aggregate else [issue.dump() for issue in self.issues]
+        return output
+
+    def __str__(self) -> str:
+        dumped = self.dump(aggregate=True)
+        lines: list[str] = []
+        for key, value in dumped.items():
+            if key in ["name", "error_messages", "issues"]:
+                continue
+            lines.append(f"{key}: {value}")
+        return f"{self.name.title()}: {', '.join(lines)}"
 
 
 @dataclass
@@ -71,6 +91,14 @@ class UploadResultIDs(UploadResultCore):
     success: list[str] = field(default_factory=list)
     failed: list[str] = field(default_factory=list)
 
+    def dump(self, aggregate: bool = True) -> dict[str, Any]:
+        output = super().dump(aggregate)
+        if self.success:
+            output["success"] = len(self.success) if aggregate else self.success
+        if self.failed:
+            output["failed"] = len(self.failed) if aggregate else self.failed
+        return output
+
 
 @dataclass
 class UploadDiffsID(UploadResultCore):
@@ -84,3 +112,15 @@ class UploadDiffsID(UploadResultCore):
         result.success = self.created + self.changed + self.unchanged
         result.failed = self.failed
         return result
+
+    def dump(self, aggregate: bool = True) -> dict[str, Any]:
+        output = super().dump(aggregate)
+        if self.created:
+            output["created"] = len(self.created) if aggregate else self.created
+        if self.changed:
+            output["changed"] = len(self.changed) if aggregate else self.changed
+        if self.unchanged:
+            output["unchanged"] = len(self.unchanged) if aggregate else self.unchanged
+        if self.failed:
+            output["failed"] = len(self.failed) if aggregate else self.failed
+        return output

cognite/neat/workflows/migration/steps.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from ruamel.yaml import YAML
+import yaml
 
 # Migration mapping
 
@@ -62,9 +62,7 @@ step_rename_mapping = {
 
 def rename_workflow_steps(workflow_path: Path, dry_run: bool) -> str:
     # Load the YAML file
-    yaml = YAML()
-    with workflow_path.open() as file:
-        data = yaml.load(file)
+    data = yaml.safe_load(workflow_path.read_text())
     # Replace old step names with new ones
     for step in data["steps"]:
         if step["method"] in step_rename_mapping:
@@ -73,7 +71,7 @@ def rename_workflow_steps(workflow_path: Path, dry_run: bool) -> str:
 
     # Save the updated YAML file
     with workflow_path.open("w") as file:
-        yaml.dump(data, file)
+        yaml.safe_dump(data, file)
     return "ok"
 
 

cognite/neat/workflows/steps/lib/current/rules_exporter.py

@@ -95,7 +95,7 @@ class DeleteDataModelFromCDF(Step):
         report_lines = ["# Data Model Deletion from CDF\n\n"]
         errors = []
         for result in dms_exporter.delete_from_cdf(rules=input_rules, client=cdf_client, dry_run=dry_run):
-            report_lines.append(result.as_report_str())
+            report_lines.append(str(result))
             errors.extend(result.error_messages)
 
         report_lines.append("\n\n# ERRORS\n\n")
@@ -216,8 +216,8 @@ class RulesToDMS(Step):
 
         report_lines = ["# DMS Schema Export to CDF\n\n"]
         errors = []
-        for result in dms_exporter.export_to_cdf(rules=input_rules, client=cdf_client, dry_run=dry_run):
-            report_lines.append(result.as_report_str())
+        for result in dms_exporter.export_to_cdf_iterable(rules=input_rules, client=cdf_client, dry_run=dry_run):
+            report_lines.append(str(result))
             errors.extend(result.error_messages)
 
         report_lines.append("\n\n# ERRORS\n\n")
@@ -555,8 +555,8 @@ class RulesToCDFTransformations(Step):
 
         report_lines = ["# DMS Schema Export to CDF\n\n"]
         errors = []
-        for result in dms_exporter.export_to_cdf(rules=input_rules, client=cdf_client, dry_run=dry_run):
-            report_lines.append(result.as_report_str())
+        for result in dms_exporter.export_to_cdf_iterable(rules=input_rules, client=cdf_client, dry_run=dry_run):
+            report_lines.append(str(result))
             errors.extend(result.error_messages)
 
         report_lines.append("\n\n# ERRORS\n\n")

{cognite_neat-0.82.3.dist-info → cognite_neat-0.84.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.82.3
+Version: 0.84.0
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0
@@ -17,15 +17,16 @@ Provides-Extra: docs
 Provides-Extra: google
 Provides-Extra: graphql
 Provides-Extra: oxi
+Provides-Extra: service
 Requires-Dist: PyYAML
 Requires-Dist: backports.strenum (>=1.2,<2.0) ; python_version < "3.11"
-Requires-Dist: cognite-sdk (>=7.37.0,<8.0.0)
+Requires-Dist: cognite-sdk (>=7.52.0,<8.0.0)
 Requires-Dist: deepdiff
 Requires-Dist: exceptiongroup (>=1.1.3,<2.0.0) ; python_version < "3.11"
-Requires-Dist: fastapi (>=0,<1)
-Requires-Dist: google-api-python-client ; extra == "google" or extra == "all"
-Requires-Dist: google-auth-oauthlib ; extra == "google" or extra == "all"
-Requires-Dist: gspread ; extra == "google" or extra == "all"
+Requires-Dist: fastapi (>=0,<1) ; extra == "service" or extra == "all"
+Requires-Dist: google-api-python-client ; extra == "google"
+Requires-Dist: google-auth-oauthlib ; extra == "google"
+Requires-Dist: gspread ; extra == "google"
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0) ; extra == "graphql" or extra == "all"
 Requires-Dist: mkdocs ; extra == "docs"
 Requires-Dist: mkdocs-autorefs (>=0.5.0,<0.6.0) ; extra == "docs"
@@ -39,18 +40,17 @@ Requires-Dist: mkdocstrings[python] ; extra == "docs"
 Requires-Dist: openpyxl
 Requires-Dist: oxrdflib[oxigraph] (>=0.3.3,<0.4.0) ; extra == "oxi" or extra == "all"
 Requires-Dist: pandas
-Requires-Dist: prometheus-client (>=0.20.0,<0.21.0)
+Requires-Dist: prometheus-client (>=0,<1) ; extra == "service" or extra == "all"
 Requires-Dist: pydantic (>=2,<3)
 Requires-Dist: pymdown-extensions ; extra == "docs"
-Requires-Dist: pyoxigraph (==0.3.19)
-Requires-Dist: python-multipart (>=0,<1)
+Requires-Dist: pyoxigraph (==0.3.19) ; extra == "oxi" or extra == "all"
 Requires-Dist: rdflib
 Requires-Dist: requests
-Requires-Dist: schedule (>=1,<2)
+Requires-Dist: schedule (>=1,<2) ; extra == "service" or extra == "all"
 Requires-Dist: tomli (>=2.0.1,<3.0.0) ; python_version < "3.11"
 Requires-Dist: typing_extensions (>=4.8,<5.0) ; python_version < "3.11"
 Requires-Dist: urllib3 (>=2,<3)
-Requires-Dist: uvicorn[standard] (>=0,<1)
+Requires-Dist: uvicorn[standard] (>=0,<1) ; extra == "service" or extra == "all"
 Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
 Project-URL: Repository, https://github.com/cognitedata/neat
 Description-Content-Type: text/markdown

{cognite_neat-0.82.3.dist-info → cognite_neat-0.84.0.dist-info}/RECORD

@@ -1,5 +1,6 @@
 cognite/neat/__init__.py,sha256=v-rRiDOgZ3sQSMQKq0vgUQZvpeOkoHFXissAx6Ktg84,61
-cognite/neat/_version.py,sha256=3x2wKMbffUEEJlJ3cDVbyt1M_iEH-TBgkPoWxClY4WU,23
+cognite/neat/_shared.py,sha256=afQiTM0SvIKqeBRTvpfwwIvZL7QMQevt4F7lqRagAFg,968
+cognite/neat/_version.py,sha256=KIiYae8LGfAm2z2vAqeoEPgeYngLtYMZnQCGOUKk3Nk,23
 cognite/neat/app/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/app/api/asgi/metrics.py,sha256=nxFy7L5cChTI0a-zkCiJ59Aq8yLuIJp5c9Dg0wRXtV0,152
 cognite/neat/app/api/configuration.py,sha256=2U5M6M252swvQPQyooA1EBzFUZNtcTmuSaywfJDgckM,4232
@@ -125,9 +126,9 @@ cognite/neat/legacy/rules/analysis.py,sha256=-PWaFqzvMlUA6nrZissMvoQm8xwFqniIDAp
 cognite/neat/legacy/rules/examples/Rules-Nordic44-to-TNT.xlsx,sha256=pT2skrX3uWZtw-HIfIipVuPVr3bgb8zOAIqkguVjhCI,58987
 cognite/neat/legacy/rules/examples/Rules-Nordic44-to-graphql.xlsx,sha256=eo6K177Xrz0CKyUFsbsGF8qZJsJI1Qf1JrzozdbmkeU,80226
 cognite/neat/legacy/rules/examples/__init__.py,sha256=rLVPMLdcxpiW6LzfIgJodDKXfpdoYbD4Q60nmK_wAqU,858
-cognite/neat/legacy/rules/examples/power-grid-containers.yaml,sha256=5Q_TCM8YuLOEOxoCWqosvYJeB4NYBA1EqlE7pyo0AOA,2598
+cognite/neat/legacy/rules/examples/power-grid-containers.yaml,sha256=dgAQ5KYT9_FHwsxPapvm00K4XJtKMbGmTs4-KnfpjDg,2782
 cognite/neat/legacy/rules/examples/power-grid-example.xlsx,sha256=gUklgEhawhJ-EquwZpNp7dFvrLGAOzbKIGh8hTObxMg,77055
-cognite/neat/legacy/rules/examples/power-grid-model.yaml,sha256=ttCtAuQYrOkWBHttQo0w5WHWpzbnM0Z7MxG_ubtwTh0,5567
+cognite/neat/legacy/rules/examples/power-grid-model.yaml,sha256=UBtwOBUkdSfkuNOOJhnOdpdPHx3Agx_BZHPMjMmTEU0,5767
 cognite/neat/legacy/rules/examples/rules-template.xlsx,sha256=buyxUbY5N5H52LgKST-D2FVvpjEMW4jC2chAHToknNI,75865
 cognite/neat/legacy/rules/examples/sheet2cdf-transformation-rules.xlsx,sha256=_a93q8Yu7nv7Bq3I__RHD7pcdbfrv7urlp_AdgP2o3o,52433
 cognite/neat/legacy/rules/examples/skos-rules.xlsx,sha256=XjdBpbfUrJnniTCCQLOBW-8OmJyGpIT1Ig-DHQG3x04,26008
@@ -187,8 +188,8 @@ cognite/neat/rules/examples/__init__.py,sha256=nxIwueAcHgZhkYriGxnDLQmIyiT8PByPH
 cognite/neat/rules/examples/wind-energy.owl,sha256=NuomCA9FuuLF0JlSuG3OKqD4VBcHgSjDKFLV17G1zV8,65934
 cognite/neat/rules/exceptions.py,sha256=YLnsbXXJdDSr_szQoioEtOdqDV8PR7RdQjpMP2SWeCs,123868
 cognite/neat/rules/exporters/__init__.py,sha256=Gn3CjkVKHJF9Po1ZPH4wAJ-sRW9up7b2CpXm-eReV3Q,413
-cognite/neat/rules/exporters/_base.py,sha256=3eLIzZ3ddVVQGV7qQn-hKV0U_lPMBQzC4AblTKYzaEA,1535
-cognite/neat/rules/exporters/_rules2dms.py,sha256=e9veycUKfZ5NFDC_ar08hNd-LWrZAva_bRFa9VfI8AA,13725
+cognite/neat/rules/exporters/_base.py,sha256=TkdpmliKjKVDITBAE4ySq_Zc8edFDQzHkbvHll4ODkg,1763
+cognite/neat/rules/exporters/_rules2dms.py,sha256=xnmq4FbMAvYVtJzDM-wwTRwVq_t3XWf8ffmhpE27BSk,14547
 cognite/neat/rules/exporters/_rules2excel.py,sha256=HvUdXYHxfLMijYWdTnfqCsw3Izf8S-XDSve-2ZbqF8Y,14248
 cognite/neat/rules/exporters/_rules2ontology.py,sha256=Od53uLdcC2Q7UiF5PA2P0gw3O14eTD3MeJ1-trd64ZM,20388
 cognite/neat/rules/exporters/_rules2yaml.py,sha256=GA8eUYRxUfIU6IMvlyGO5JidkOD5eUKSbH3qAiFiaCg,3026
@@ -236,7 +237,7 @@ cognite/neat/rules/models/dms/_converter.py,sha256=QaJT0z0Yo4gkG9tHPZkU8idF8PK7c
 cognite/neat/rules/models/dms/_exporter.py,sha256=DRL5ahcBEdBltHLA2rzLyUfqG5SZkqINkqmYGDkQVg0,24785
 cognite/neat/rules/models/dms/_rules.py,sha256=XTIEWG49VjNs_bICGlgMd6uk4hseY1H6UVe2KMfHAWQ,16152
 cognite/neat/rules/models/dms/_rules_input.py,sha256=apDDTQll9UAyYL5gS2vDxHsujWrGBilTp7lK2kzJWO8,13467
-cognite/neat/rules/models/dms/_schema.py,sha256=byMG67i80a4sSQS_0k8YGrDvh7whio4iLbmPEIy_P44,49514
+cognite/neat/rules/models/dms/_schema.py,sha256=balykJXJy93HJzBZN-KBYrRJJvGsaEtQeX7-z0tbX7w,49566
 cognite/neat/rules/models/dms/_serializer.py,sha256=iqp2zyyf8jEcU-R3PERuN8nu248xIqyxiWj4owAn92g,6406
 cognite/neat/rules/models/dms/_validation.py,sha256=5mk9L99FSwC8Ok7weEjnFJ_OZnmqMWUc6XFMTfkqfDw,14549
 cognite/neat/rules/models/domain.py,sha256=wZ-DeIPFnacbNlxSrRuLzUpnhHdTpzNc22z0sDfisi4,2880
@@ -253,14 +254,14 @@ cognite/neat/utils/auxiliary.py,sha256=E2-YtddzScvN7l7j0kNYIMlfqIUT9NWMqLpcJYPK4
 cognite/neat/utils/cdf.py,sha256=piRx-6GRz4cCfBZD5rU0OM6ixQ3cj5TMzI0yCYUveR8,2422
 cognite/neat/utils/cdf_classes.py,sha256=NEmz5UprBlqfqZnqJkRk5xjSpzazwHbhcWsMH_GNxP8,5831
 cognite/neat/utils/cdf_loaders/__init__.py,sha256=s2aPR5XLo6WZ0ybstAJlcGFYkA7CyHW1XO-NYpL0V6o,483
-cognite/neat/utils/cdf_loaders/_base.py,sha256=j85HINIx4poGD-0dN3JPHTqjOS1XhCxv7G9s1gC1GBo,2057
-cognite/neat/utils/cdf_loaders/_data_modeling.py,sha256=RoIj2cL_j3vP6e4BlCExZU8d96p3dLitU9nsIIyesLc,11336
+cognite/neat/utils/cdf_loaders/_base.py,sha256=ryNC_AMXIESWXuTVJ-02L-HSVSpD6V49XdLTRYeFg70,1764
+cognite/neat/utils/cdf_loaders/_data_modeling.py,sha256=-owgamlYnB4-hnvsvWiGPFcnvkKs0EzrrWmIKhnvBpI,11359
 cognite/neat/utils/cdf_loaders/_ingestion.py,sha256=_2G8BObvpZ-vTHaCMAW_-ckHBLY96Hbg27uKAvpCaWk,6303
 cognite/neat/utils/cdf_loaders/data_classes.py,sha256=0apspfwVlFltYOZfmk_PNknS3Z3SCxVr3kkvXH0bfPA,3844
 cognite/neat/utils/exceptions.py,sha256=-w4cAcvcoWLf-_ZwAl7QV_NysfqtQzIOd1Ti-mpxJgM,981
 cognite/neat/utils/spreadsheet.py,sha256=LI0c7dlW0zXHkHw0NvB-gg6Df6cDcE3FbiaHBYLXdzQ,2714
 cognite/neat/utils/text.py,sha256=4bg1_Q0lg7KsoxaDOvXrVyeY78BJN8i-27BlyDzUCls,3082
-cognite/neat/utils/upload.py,sha256=XaAKqyMhz6qXbUrttGNIXZxFRPJvrnbMpDRF8GEiK2g,2707
+cognite/neat/utils/upload.py,sha256=opAB8oDtpgcrugcjbUg0tjGqtFUnAS7zixtLiRYZ3TA,5084
 cognite/neat/utils/utils.py,sha256=1LEwR8gpHw_6pvEeLkW_cDU_lUun4qSsw_Rr3JsKwgA,14172
 cognite/neat/utils/xml.py,sha256=ppLT3lQKVp8wOP-m8-tFY8uB2P4R76l7R_-kUtsABng,992
 cognite/neat/workflows/__init__.py,sha256=oiKub_U9f5cA0I1nKl5dFkR4BD8_6Be9eMzQ_50PwP0,396
@@ -279,7 +280,7 @@ cognite/neat/workflows/examples/Visualize_Data_Model_Using_Mock_Graph/workflow.y
 cognite/neat/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml,sha256=yWVL-NHghKtiNV2kpEX674MJwWqhOUn3j2ZOJiJbprE,2579
 cognite/neat/workflows/manager.py,sha256=CtxnabAmZq4APd5XA6FTBlytP3WmlbKNGjNoR04uAQ4,14102
 cognite/neat/workflows/migration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cognite/neat/workflows/migration/steps.py,sha256=4YCPWwfEqzIgLN4lRNx8Y-nj14ptNxH8-SdLY2HJbvE,3992
+cognite/neat/workflows/migration/steps.py,sha256=YjwSDrf6W8pvKzh0vnkyA2QXJDGoYrzJ6VHNCy2oPtc,3945
 cognite/neat/workflows/migration/wf_manifests.py,sha256=TIJ9q_sbZ1LVJfUYVk8VRYcxrRHlwyktHRag0OJcVrE,1556
 cognite/neat/workflows/model.py,sha256=LQY7abYnz3CUEIlIEqoj0Eo6Q8yQukTQ0S_sPststCA,6570
 cognite/neat/workflows/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -289,7 +290,7 @@ cognite/neat/workflows/steps/lib/current/__init__.py,sha256=c22IznGdCSNCpXCi_yon
 cognite/neat/workflows/steps/lib/current/graph_extractor.py,sha256=vW9UpJScx5dFVCSairpOdWRdBdLpkCt2kNh6litbF0o,5161
 cognite/neat/workflows/steps/lib/current/graph_loader.py,sha256=HfGg1HRZhbV58TFu89FTjKeUxGsbCYLeFJIQFDN_pQM,2341
 cognite/neat/workflows/steps/lib/current/graph_store.py,sha256=r7VTxdaz8jJQU7FJbnRDMxvEYbSAZFNMABhPyfNwiFk,6295
-cognite/neat/workflows/steps/lib/current/rules_exporter.py,sha256=iFwzDWgUDBSPajaNAcXvu9pVw-jX66upvwyMXyTq7SE,23822
+cognite/neat/workflows/steps/lib/current/rules_exporter.py,sha256=1SKBYDnyDr975oMr9aOQVHExHjPQ0-whDDBNpTQ4K1A,23807
 cognite/neat/workflows/steps/lib/current/rules_importer.py,sha256=23dxamGHSbJoG7qUqL4KiGznSmDTVHw1EfqhLCeAwDM,14695
 cognite/neat/workflows/steps/lib/current/rules_validator.py,sha256=LwF9lXlnuPOxDSsOMMTHBi2BHc5rk08IF5zahS9yQuw,4844
 cognite/neat/workflows/steps/lib/io/__init__.py,sha256=k7IPbIq3ey19oRc5sA_15F99-O6dxzqbm1LihGRRo5A,32
@@ -307,8 +308,8 @@ cognite/neat/workflows/steps_registry.py,sha256=fkTX14ZA7_gkUYfWIlx7A1XbCidvqR23
 cognite/neat/workflows/tasks.py,sha256=dqlJwKAb0jlkl7abbY8RRz3m7MT4SK8-7cntMWkOYjw,788
 cognite/neat/workflows/triggers.py,sha256=_BLNplzoz0iic367u1mhHMHiUrCwP-SLK6_CZzfODX0,7071
 cognite/neat/workflows/utils.py,sha256=gKdy3RLG7ctRhbCRwaDIWpL9Mi98zm56-d4jfHDqP1E,453
-cognite_neat-0.82.3.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-0.82.3.dist-info/METADATA,sha256=vFK-g0VZN6HPCgAo_yEgu5XPKpAIbq1Gyhj2Tz8091w,9290
-cognite_neat-0.82.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cognite_neat-0.82.3.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
-cognite_neat-0.82.3.dist-info/RECORD,,
+cognite_neat-0.84.0.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-0.84.0.dist-info/METADATA,sha256=zZPlQ6y1y-rn-rkHusErmEcs7iMxe--hFR_DBM8wYao,9400
+cognite_neat-0.84.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cognite_neat-0.84.0.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
+cognite_neat-0.84.0.dist-info/RECORD,,