cognite-neat 0.75.5__tar.gz → 0.75.7__tar.gz

{cognite_neat-0.75.5 → cognite_neat-0.75.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.75.5
+Version: 0.75.7
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.75.7/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.75.7"

{cognite_neat-0.75.5 → cognite_neat-0.75.7}/cognite/neat/rules/exporters/_models.py

@@ -30,6 +30,7 @@ class UploadResult(UploadResultCore):
     skipped: int = 0
     failed_created: int = 0
     failed_changed: int = 0
+    failed_deleted: int = 0
     error_messages: list[str] = field(default_factory=list)
 
     @property
@@ -38,7 +39,7 @@ class UploadResult(UploadResultCore):
 
     @property
     def failed(self) -> int:
-        return self.failed_created + self.failed_changed
+        return self.failed_created + self.failed_changed + self.failed_deleted
 
     def as_report_str(self) -> str:
         line = []
@@ -50,9 +51,13 @@ class UploadResult(UploadResultCore):
             line.append(f"skipped {self.skipped}")
         if self.unchanged:
             line.append(f"unchanged {self.unchanged}")
+        if self.deleted:
+            line.append(f"deleted {self.deleted}")
         if self.failed_created:
             line.append(f"failed to create {self.failed_created}")
         if self.failed_changed:
             line.append(f"failed to update {self.failed_changed}")
+        if self.failed_deleted:
+            line.append(f"failed to delete {self.failed_deleted}")
 
         return f"{self.name.title()}: {', '.join(line)}"

{cognite_neat-0.75.5 → cognite_neat-0.75.7}/cognite/neat/rules/exporters/_rules2dms.py

@@ -27,7 +27,7 @@ from cognite.neat.utils.cdf_loaders import (
 from ._base import CDFExporter
 from ._models import UploadResult
 
-Component: TypeAlias = Literal["all", "spaces", "data_models", "views", "containers"]
+Component: TypeAlias = Literal["all", "spaces", "data_models", "views", "containers", "node_types"]
 
 
 class DMSExporter(CDFExporter[DMSSchema]):
@@ -99,7 +99,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
         if "all" in self.export_components:
             exclude = set()
         else:
-            exclude = {"spaces", "data_models", "views", "containers"} - self.export_components
+            exclude = {"spaces", "data_models", "views", "containers", "node_types"} - self.export_components
         return exclude
 
     def export(self, rules: Rules) -> DMSSchema:
@@ -118,12 +118,12 @@ class DMSExporter(CDFExporter[DMSSchema]):
             return dms_rules.as_schema(self.export_pipeline, self.instance_space)
 
         # This is an extension of an existing model.
-        reference_rules = cast(DMSRules, dms_rules.reference).copy(deep=True)
+        reference_rules = cast(DMSRules, dms_rules.reference).model_copy(deep=True)
         reference_schema = reference_rules.as_schema(self.export_pipeline)
 
         # Todo Move this to an appropriate location
         # Merging Reference with User Rules
-        combined_rules = dms_rules.copy(deep=True)
+        combined_rules = dms_rules.model_copy(deep=True)
         existing_containers = {container.class_ for container in combined_rules.containers or []}
         if combined_rules.containers is None:
             combined_rules.containers = SheetList[DMSContainer](data=[])
@@ -151,21 +151,55 @@ class DMSExporter(CDFExporter[DMSSchema]):
             schema.frozen_ids.update(set(reference_schema.node_types.as_ids()))
         return schema
 
+    def delete_from_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
+        schema, to_export = self._prepare_schema_and_exporters(rules, client)
+
+        # we need to reverse order in which we are picking up the items to delete
+        # as they are sorted in the order of creation and we need to delete them in reverse order
+        for all_items, loader in reversed(to_export):
+            all_item_ids = loader.get_ids(all_items)
+            skipped = sum(1 for item_id in all_item_ids if item_id in schema.frozen_ids)
+            item_ids = [item_id for item_id in all_item_ids if item_id not in schema.frozen_ids]
+            cdf_items = loader.retrieve(item_ids)
+            cdf_item_by_id = {loader.get_id(item): item for item in cdf_items}
+            items = [item for item in all_items if loader.get_id(item) in item_ids]
+            to_delete = []
+
+            for item in items:
+                if (
+                    isinstance(loader, DataModelingLoader)
+                    and self.include_space is not None
+                    and not loader.in_space(item, self.include_space)
+                ):
+                    continue
+
+                cdf_item = cdf_item_by_id.get(loader.get_id(item))
+                if cdf_item:
+                    to_delete.append(cdf_item)
+
+            deleted = len(to_delete)
+            failed_deleted = 0
+
+            error_messages: list[str] = []
+            if not dry_run:
+                if to_delete:
+                    try:
+                        loader.delete(to_delete)
+                    except CogniteAPIError as e:
+                        failed_deleted = len(e.failed) + len(e.unknown)
+                        deleted -= failed_deleted
+                        error_messages.append(f"Failed delete: {e.message}")
+
+            yield UploadResult(
+                name=loader.resource_name,
+                deleted=deleted,
+                skipped=skipped,
+                failed_deleted=failed_deleted,
+                error_messages=error_messages,
+            )
+
     def export_to_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
-        schema = self.export(rules)
-        to_export: list[tuple[CogniteResourceList, ResourceLoader]] = []
-        if self.export_components.intersection({"all", "spaces"}):
-            to_export.append((schema.spaces, SpaceLoader(client)))
-        if self.export_components.intersection({"all", "containers"}):
-            to_export.append((schema.containers, ContainerLoader(client)))
-        if self.export_components.intersection({"all", "views"}):
-            to_export.append((schema.views, ViewLoader(client, self.existing_handling)))
-        if self.export_components.intersection({"all", "data_models"}):
-            to_export.append((schema.data_models, DataModelLoader(client)))
-        if isinstance(schema, PipelineSchema):
-            to_export.append((schema.databases, RawDatabaseLoader(client)))
-            to_export.append((schema.raw_tables, RawTableLoader(client)))
-            to_export.append((schema.transformations, TransformationLoader(client)))
+        schema, to_export = self._prepare_schema_and_exporters(rules, client)
 
         # The conversion from DMS to GraphQL does not seem to be triggered even if the views
         # are changed. This is a workaround to force the conversion.
@@ -254,3 +288,22 @@ class DMSExporter(CDFExporter[DMSSchema]):
 
             if loader.resource_name == "views" and (created or changed) and not redeploy_data_model:
                 redeploy_data_model = True
+
+    def _prepare_schema_and_exporters(
+        self, rules, client
+    ) -> tuple[DMSSchema, list[tuple[CogniteResourceList, ResourceLoader]]]:
+        schema = self.export(rules)
+        to_export: list[tuple[CogniteResourceList, ResourceLoader]] = []
+        if self.export_components.intersection({"all", "spaces"}):
+            to_export.append((schema.spaces, SpaceLoader(client)))
+        if self.export_components.intersection({"all", "containers"}):
+            to_export.append((schema.containers, ContainerLoader(client)))
+        if self.export_components.intersection({"all", "views"}):
+            to_export.append((schema.views, ViewLoader(client, self.existing_handling)))
+        if self.export_components.intersection({"all", "data_models"}):
+            to_export.append((schema.data_models, DataModelLoader(client)))
+        if isinstance(schema, PipelineSchema):
+            to_export.append((schema.databases, RawDatabaseLoader(client)))
+            to_export.append((schema.raw_tables, RawTableLoader(client)))
+            to_export.append((schema.transformations, TransformationLoader(client)))
+        return schema, to_export

{cognite_neat-0.75.5 → cognite_neat-0.75.7}/cognite/neat/rules/exporters/_rules2excel.py

@@ -29,6 +29,11 @@ class ExcelExporter(BaseExporter[Workbook]):
         new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
             rules have 'is_reference' set. If provided, the model ID will be used to automatically create the
             new metadata sheet in the Excel file.
+        is_reference: If True, the rules are considered to be a reference model. The exported Excel file will
+            then contain empty sheets for the main rules and this data model will be dumped to the reference sheets.
+            This is useful when you are building a solution model based on an Enterprise model, then the
+            Enterprise model will serve as the reference model. It is also useful when you are extending an existing
+            model, then the existing model will serve as the reference model. Defaults to False.
 
     The following styles are available:
 
@@ -54,6 +59,7 @@ class ExcelExporter(BaseExporter[Workbook]):
         styling: Style = "default",
         output_role: RoleTypes | None = None,
         new_model_id: tuple[str, str, str] | None = None,
+        is_reference: bool = False,
     ):
         if styling not in self.style_options:
             raise ValueError(f"Invalid styling: {styling}. Valid options are {self.style_options}")
@@ -61,6 +67,7 @@ class ExcelExporter(BaseExporter[Workbook]):
         self._styling_level = self.style_options.index(styling)
         self.output_role = output_role
         self.new_model_id = new_model_id
+        self.is_reference = is_reference
 
     def export_to_file(self, rules: Rules, filepath: Path) -> None:
         """Exports transformation rules to excel file."""
@@ -79,7 +86,7 @@ class ExcelExporter(BaseExporter[Workbook]):
 
         dumped_rules: dict[str, Any]
         dumped_reference_rules: dict[str, Any] | None = None
-        if rules.is_reference:
+        if self.is_reference:
             # Writes empty reference sheets
             dumped_rules = {
                 "Metadata": self._create_metadata_sheet_user_rules(rules),

{cognite_neat-0.75.5 → cognite_neat-0.75.7}/cognite/neat/rules/importers/_base.py

@@ -20,21 +20,18 @@ class BaseImporter(ABC):
     """
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
     def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None, is_reference: bool = False
+        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     @abstractmethod
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         """
         Creates `Rules` object from the data for target role.
@@ -48,7 +45,6 @@ class BaseImporter(ABC):
         issues: IssueList,
         errors: Literal["raise", "continue"] = "continue",
         role: RoleTypes | None = None,
-        is_reference: bool = False,
     ) -> tuple[Rules | None, IssueList] | Rules:
         """Converts the rules to the output format."""
 
@@ -61,9 +57,6 @@ class BaseImporter(ABC):
         else:
             raise NotImplementedError(f"Role {role} is not supported for {type(rules).__name__} rules")
 
-        if is_reference:
-            output.is_reference = True
-
         if errors == "raise":
             return output
         else:

{cognite_neat-0.75.5 → cognite_neat-0.75.7}/cognite/neat/rules/importers/_dms2rules.py

@@ -82,20 +82,17 @@ class DMSImporter(BaseImporter):
         return cls(DMSSchema.from_zip(zip_file))
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
     def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None, is_reference: bool = False
+        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         if role is RoleTypes.domain_expert:
             raise ValueError(f"Role {role} is not supported for DMSImporter")
@@ -210,7 +207,6 @@ class DMSImporter(BaseImporter):
                 data=[DMSContainer.from_container(container) for container in self.schema.containers]
             ),
             views=SheetList[DMSView](data=[DMSView.from_view(view, data_model_view_ids) for view in self.schema.views]),
-            is_reference=is_reference,
         )
         output_rules: Rules
         if role is RoleTypes.information_architect:

{cognite_neat-0.75.5 → cognite_neat-0.75.7}/cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py

@@ -119,20 +119,17 @@ class DTDLImporter(BaseImporter):
         return cls(items, zip_file.stem, read_issues=issues)
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
     def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None, is_reference: bool = False
+        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         converter = _DTDLConverter(self._read_issues)
 
@@ -161,4 +158,4 @@ class DTDLImporter(BaseImporter):
             else:
                 raise converter.issues.as_errors()
 
-        return self._to_output(rules, converter.issues, errors, role, is_reference)
+        return self._to_output(rules, converter.issues, errors, role)

{cognite_neat-0.75.5 → cognite_neat-0.75.7}/cognite/neat/rules/importers/_owl2rules/_owl2rules.py

@@ -43,20 +43,17 @@ class OWLImporter(BaseImporter):
         self.make_compliant = make_compliant
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
     def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None, is_reference: bool = False
+        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         graph = Graph()
         try:
@@ -82,7 +79,7 @@ class OWLImporter(BaseImporter):
             components = make_components_compliant(components)
 
         rules = InformationRules.model_validate(components)
-        return self._to_output(rules, IssueList(), errors, role, is_reference)
+        return self._to_output(rules, IssueList(), errors, role)
 
 
 def make_components_compliant(components: dict) -> dict:

{cognite_neat-0.75.5 → cognite_neat-0.75.7}/cognite/neat/rules/importers/_spreadsheet2rules.py

@@ -152,7 +152,7 @@ class ExcelImporter(BaseImporter):
         self.filepath = filepath
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
@@ -160,29 +160,23 @@ class ExcelImporter(BaseImporter):
         self,
         errors: Literal["continue"] = "continue",
         role: RoleTypes | None = None,
-        is_reference: bool = False,
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         issue_list = IssueList(title=f"'{self.filepath.name}'")
         if not self.filepath.exists():
             issue_list.append(issues.spreadsheet_file.SpreadsheetNotFoundError(self.filepath))
             return self._return_or_raise(issue_list, errors)
 
-        user_result: ReadResult | None = None
-        if not is_reference:
-            user_result = SpreadsheetReader(issue_list, is_reference=False).read(self.filepath)
-            if user_result is None or issue_list.has_errors:
-                return self._return_or_raise(issue_list, errors)
+        user_result = SpreadsheetReader(issue_list, is_reference=False).read(self.filepath)
+        if user_result is None or issue_list.has_errors:
+            return self._return_or_raise(issue_list, errors)
 
         reference_result: ReadResult | None = None
-        if is_reference or (
+        if (
             user_result
             and user_result.role != RoleTypes.domain_expert
             and user_result.schema == SchemaCompleteness.extended
@@ -230,7 +224,6 @@ class ExcelImporter(BaseImporter):
             issue_list,
             errors=errors,
             role=role,
-            is_reference=is_reference,
         )
 
     @classmethod
@@ -246,20 +239,17 @@ class GoogleSheetImporter(BaseImporter):
         self.skiprows = skiprows
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
     def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None, is_reference: bool = False
+        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         local_import("gspread", "google")
         import gspread  # type: ignore[import]
@@ -284,4 +274,4 @@ class GoogleSheetImporter(BaseImporter):
         else:
             raise ValueError(f"Role {role} is not valid.")
 
-        return self._to_output(output, IssueList(), errors=errors, role=role, is_reference=is_reference)
+        return self._to_output(output, IssueList(), errors=errors, role=role)

{cognite_neat-0.75.5 → cognite_neat-0.75.7}/cognite/neat/rules/importers/_yaml2rules.py

@@ -45,20 +45,17 @@ class YAMLImporter(BaseImporter):
         return cls(yaml.safe_load(filepath.read_text()), filepaths=[filepath])
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
     def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None, is_reference: bool = False
+        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         if any(issue for issue in self._read_issues if isinstance(issue, NeatValidationError)) or not self.raw_data:
             if errors == "raise":
@@ -107,4 +104,4 @@ class YAMLImporter(BaseImporter):
             else:
                 raise issue_list.as_errors()
 
-        return self._to_output(rules, issue_list, errors, role, is_reference)
+        return self._to_output(rules, issue_list, errors, role)