cognite-neat 0.109.0__py3-none-any.whl → 0.109.2__py3-none-any.whl

cognite/neat/_alpha.py

@@ -0,0 +1,14 @@
+import warnings
+
+
+class AlphaWarning(UserWarning):
+    def __init__(self, feature_name: str):
+        super().__init__(f"Alpha feature '{feature_name}' is subject to change without notice")
+
+    def warn(self) -> None:
+        warnings.warn(self, stacklevel=2)
+
+
+class AlphaFlags:
+    manual_rules_edit = AlphaWarning("enable_manual_edit")
+    same_space_properties_only_export = AlphaWarning("same-space-properties-only")

cognite/neat/_rules/exporters/_rules2excel.py

@@ -22,6 +22,7 @@ from cognite.neat._rules.models import (
 from cognite.neat._rules.models.dms import DMSMetadata
 from cognite.neat._rules.models.information import InformationMetadata
 from cognite.neat._rules.models.information._rules import InformationRules
+from cognite.neat._utils.spreadsheet import find_column_with_value
 
 from ._base import BaseExporter
 
@@ -63,12 +64,22 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
     style_options = get_args(Style)
     dump_options = get_args(DumpOptions)
 
+    _internal_columns: ClassVar[list[str]] = [
+        "physical",
+        "logical",
+        "conceptual",
+        "Neat ID",
+    ]
+
     def __init__(
         self,
         styling: Style = "default",
         new_model_id: tuple[str, str] | None = None,
         sheet_prefix: str | None = None,
         reference_rules_with_prefix: tuple[VerifiedRules, str] | None = None,
+        add_empty_rows: bool = False,
+        hide_internal_columns: bool = True,
+        include_properties: Literal["same-space", "all"] = "all",
     ):
         self.sheet_prefix = sheet_prefix or ""
         if styling not in self.style_options:
@@ -77,6 +88,9 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
         self._styling_level = self.style_options.index(styling)
         self.new_model_id = new_model_id
         self.reference_rules_with_prefix = reference_rules_with_prefix
+        self.add_empty_rows = add_empty_rows
+        self.hide_internal_columns = hide_internal_columns
+        self.include_properties = include_properties
 
     @property
     def description(self) -> str:
@@ -112,6 +126,16 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
         if self._styling_level > 0:
             self._adjust_column_widths(workbook)
 
+        if self.hide_internal_columns:
+            for sheet in workbook.sheetnames:
+                if sheet.lower() == "metadata":
+                    continue
+                ws = workbook[sheet]
+                for col in self._internal_columns:
+                    column_letter = find_column_with_value(ws, col)
+                    if column_letter:
+                        ws.column_dimensions[column_letter].hidden = True
+
         return workbook
 
     def _write_sheets(
@@ -149,13 +173,19 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
                 is_properties = sheet_name == "Properties"
                 is_new_class = class_ != last_class and last_class is not None
                 if self._styling_level > 2 and is_new_class and is_properties:
-                    sheet.append([""] * len(headers))
+                    if self.add_empty_rows:
+                        sheet.append([""] * len(headers))
                     for cell in sheet[sheet.max_row]:
                         cell.fill = PatternFill(fgColor=fill_color, patternType="solid")
                         side = Side(style="thin", color="000000")
                         cell.border = Border(left=side, right=side, top=side, bottom=side)
                     fill_color = next(fill_colors)
 
+                if is_properties and self.include_properties == "same-space":
+                    space = class_.split(":")[0] if ":" in class_ else rules.metadata.space
+                    if space != rules.metadata.space:
+                        continue
+
                 sheet.append(row)
                 if self._styling_level > 2 and is_properties:
                     for cell in sheet[sheet.max_row]:

cognite/neat/_rules/models/entities/_wrapped.py

@@ -201,3 +201,6 @@ class RawFilter(DMSFilter):
 
     def __repr__(self) -> str:
         return self.filter
+
+    def __str__(self):
+        return f"{self.name}({self.filter})"

cognite/neat/_session/_collector.py

@@ -65,7 +65,10 @@ class Collector:
         if kwargs:
             for key, value in kwargs.items():
                 event_information[key] = self._serialize_value(value)[:500]
-        self._track(command, event_information)
+
+        with suppress(RuntimeError):
+            # In case any thread issues, the tracking should not crash the program
+            self._track(command, event_information)
 
     @staticmethod
     def _serialize_value(value: Any) -> str:

cognite/neat/_session/_inspect.py

@@ -110,7 +110,7 @@ class InspectIssues:
         if issues is None and self._state.instances.store.provenance:
             last_change = self._state.instances.store.provenance[-1]
             issues = last_change.target_entity.issues
-        else:
+        elif issues is None:
             self._print("No issues found.")
             return pd.DataFrame() if return_dataframe else None
 

cognite/neat/_session/_read.py

@@ -1,8 +1,10 @@
+import warnings
 from typing import Any, Literal, cast
 
 from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
 from cognite.client.utils.useful_types import SequenceNotStr
 
+from cognite.neat._alpha import AlphaFlags
 from cognite.neat._client import NeatClient
 from cognite.neat._constants import (
     CLASSIC_CDF_NAMESPACE,
@@ -286,16 +288,26 @@ class ExcelReadAPI(BaseReadAPI):
         super().__init__(state, verbose)
         self.examples = ExcelExampleAPI(state, verbose)
 
-    def __call__(self, io: Any) -> IssueList:
+    def __call__(self, io: Any, enable_manual_edit: bool = False) -> IssueList:
         """Reads a Neat Excel Rules sheet to the graph store. The rules sheet may stem from an Information architect,
         or a DMS Architect.
 
         Args:
             io: file path to the Excel sheet
+            enable_manual_edit: If True, the user will be able to re-import rules which where edit outside NeatSession
+
+        !!! note "Manual Edit Warning"
+            This is an alpha feature and is subject to change without notice.
+            It is expected to have some limitations and may not work as expected in all cases.
         """
         reader = NeatReader.create(io)
         path = reader.materialize_path()
-        return self._state.rule_import(importers.ExcelImporter(path))
+
+        if enable_manual_edit:
+            warnings.filterwarnings("default")
+            AlphaFlags.manual_rules_edit.warn()
+
+        return self._state.rule_import(importers.ExcelImporter(path), enable_manual_edit)
 
 
 @session_class_wrapper

cognite/neat/_session/_state.py

@@ -40,8 +40,12 @@ class SessionState:
         self.instances.store.add_rules(last_entity.information)
         return issues
 
-    def rule_import(self, importer: BaseImporter) -> IssueList:
-        issues = self.rule_store.import_rules(importer, client=self.client)
+    def rule_import(self, importer: BaseImporter, enable_manual_edit: bool = False) -> IssueList:
+        issues = self.rule_store.import_rules(
+            importer,
+            client=self.client,
+            enable_manual_edit=enable_manual_edit,
+        )
         if self.rule_store.empty:
             result = "failed"
         else:

cognite/neat/_session/_to.py

@@ -6,6 +6,7 @@ from typing import Any, Literal, overload
 
 from cognite.client import data_modeling as dm
 
+from cognite.neat._alpha import AlphaFlags
 from cognite.neat._constants import COGNITE_MODELS
 from cognite.neat._graph import loaders
 from cognite.neat._rules import exporters
@@ -35,6 +36,8 @@ class ToAPI:
         self,
         io: Any,
         include_reference: bool = True,
+        include_properties: Literal["same-space", "all"] = "all",
+        add_empty_rows: bool = False,
     ) -> None:
         """Export the verified data model to Excel.
 
@@ -42,7 +45,10 @@ class ToAPI:
             io: The file path or file-like object to write the Excel file to.
             include_reference: If True, the reference data model will be included. Defaults to True.
                 Note that this only applies if you have created the data model using the
-                .to_enterprise(), .to_solution(), or .to_data_product() methods.
+                create.enterprise_model(...), create.solution_model(), or create.data_product_model() methods.
+        include_properties: The properties to include in the Excel file. Defaults to "all".
+            - "same-space": Only properties that are in the same space as the data model will be included.
+        add_empty_rows: If True, empty rows will be added between each component. Defaults to False.
 
         Example:
             Export information model to excel rules sheet
@@ -58,17 +64,17 @@ class ToAPI:
             neat = NeatSession(client)
 
             neat.read.cdf(("cdf_cdm", "CogniteCore", "v1"))
-            neat.verify()
-            neat.prepare.data_model.to_enterprise(
+            neat.create.enterprise_model(
                 data_model_id=("sp_doctrino_space", "ExtensionCore", "v1"),
                 org_name="MyOrg",
-                move_connections=True
             )
             dms_rules_file_name = "dms_rules.xlsx"
             neat.to.excel(dms_rules_file_name, include_reference=True)
             ```
         """
         reference_rules_with_prefix: tuple[VerifiedRules, str] | None = None
+        include_properties = include_properties.strip().lower()
+
         if include_reference and self._state.last_reference:
             if (
                 isinstance(self._state.last_reference.metadata, DMSMetadata)
@@ -79,7 +85,16 @@ class ToAPI:
                 prefix = "Ref"
             reference_rules_with_prefix = self._state.last_reference, prefix
 
-        exporter = exporters.ExcelExporter(styling="maximal", reference_rules_with_prefix=reference_rules_with_prefix)
+        if include_properties == "same-space":
+            warnings.filterwarnings("default")
+            AlphaFlags.same_space_properties_only_export.warn()
+
+        exporter = exporters.ExcelExporter(
+            styling="maximal",
+            reference_rules_with_prefix=reference_rules_with_prefix,
+            add_empty_rows=add_empty_rows,
+            include_properties=include_properties,  # type: ignore
+        )
         return self._state.rule_store.export_to_file(exporter, Path(io))
 
     def session(self, io: Any) -> None:

cognite/neat/_session/exceptions.py

@@ -1,7 +1,9 @@
 import functools
+import warnings
 from collections.abc import Callable
 from typing import Any
 
+from cognite.neat._alpha import AlphaWarning
 from cognite.neat._issues.errors import CDFMissingClientError, NeatImportError
 from cognite.neat._issues.errors._external import OxigraphStorageLockedError
 from cognite.neat._issues.errors._general import NeatValueError
@@ -12,9 +14,11 @@ try:
     from rich import print
     from rich.markup import escape
 
-    _PREFIX = "[bold red][ERROR][/bold red]"
+    _ERROR_PREFIX = "[bold red][ERROR][/bold red]"
+    _WARNING_PREFIX = "[bold bright_magenta][WARNING][/bold bright_magenta]"
 except ImportError:
-    _PREFIX = "[ERROR]"
+    _ERROR_PREFIX = "[ERROR]"
+    _WARNING_PREFIX = "[WARNING]"
 
     def escape(x: Any, *_: Any, **__: Any) -> Any:  # type: ignore[misc]
         return x
@@ -31,21 +35,27 @@ def _session_method_wrapper(func: Callable, cls_name: str):
     def wrapper(*args: Any, **kwargs: Any):
         _COLLECTOR.track_session_command(f"{cls_name}.{func.__name__}", *args, **kwargs)
         try:
-            return func(*args, **kwargs)
+            with warnings.catch_warnings(record=True) as w:
+                result = func(*args, **kwargs)
+                for warning in w:
+                    if isinstance(warning.message, AlphaWarning):
+                        print(f"{_WARNING_PREFIX} {warning.message}")
+
+            return result
         except NeatSessionError as e:
             action = _get_action()
-            print(f"{_PREFIX} Cannot {action}: {e}")
+            print(f"{_ERROR_PREFIX} Cannot {action}: {e}")
         except (
             CDFMissingClientError,
             NeatImportError,
             NeatValueError,
             OxigraphStorageLockedError,
         ) as e:
-            print(f"{_PREFIX} {escape(e.as_message())}")
+            print(f"{_ERROR_PREFIX} {escape(e.as_message())}")
         except ModuleNotFoundError as e:
             if e.name == "neatengine":
                 action = _get_action()
-                print(f"{_PREFIX} The functionality {action} requires the NeatEngine.")
+                print(f"{_ERROR_PREFIX} The functionality {action} requires the NeatEngine.")
             else:
                 raise e
 

cognite/neat/_store/_provenance.py

@@ -44,6 +44,7 @@ class Agent:
 CDF_AGENT = Agent(acted_on_behalf_of="UNKNOWN", id_=CDF_NAMESPACE["agent"])
 NEAT_AGENT = Agent(acted_on_behalf_of="UNKNOWN", id_=DEFAULT_NAMESPACE["agent"])
 UNKNOWN_AGENT = Agent(acted_on_behalf_of="UNKNOWN", id_=DEFAULT_NAMESPACE["unknown-agent"])
+EXTERNAL_AGENT = Agent(acted_on_behalf_of="HUMAN", id_=DEFAULT_NAMESPACE["external-agent"])
 
 
 @dataclass(frozen=True)

cognite/neat/_store/_rules_store.py

@@ -3,11 +3,11 @@ from collections import defaultdict
 from collections.abc import Callable, Hashable
 from dataclasses import dataclass
 from datetime import datetime, timezone
+from functools import partial
 from pathlib import Path
-from typing import Any
+from typing import Any, cast
 
 import rdflib
-from cognite.client import data_modeling as dm
 from rdflib import URIRef
 
 from cognite.neat._client import NeatClient
@@ -15,7 +15,7 @@ from cognite.neat._constants import DEFAULT_NAMESPACE
 from cognite.neat._graph.extractors import DMSGraphExtractor, KnowledgeGraphExtractor
 from cognite.neat._issues import IssueList, catch_issues
 from cognite.neat._issues.errors import NeatValueError
-from cognite.neat._rules._shared import ReadRules, Rules, T_VerifiedRules, VerifiedRules
+from cognite.neat._rules._shared import T_VerifiedRules, VerifiedRules
 from cognite.neat._rules.exporters import BaseExporter
 from cognite.neat._rules.exporters._base import CDFExporter, T_Export
 from cognite.neat._rules.importers import BaseImporter
@@ -23,7 +23,14 @@ from cognite.neat._rules.models import DMSRules, InformationRules
 from cognite.neat._rules.transformers import DMSToInformation, VerifiedRulesTransformer, VerifyAnyRules
 from cognite.neat._utils.upload import UploadResultList
 
-from ._provenance import UNKNOWN_AGENT, Activity, Change, Entity, Provenance
+from ._provenance import (
+    EXTERNAL_AGENT,
+    UNKNOWN_AGENT,
+    Activity,
+    Change,
+    Entity,
+    Provenance,
+)
 from .exceptions import EmptyStore, InvalidActivityInput
 
 
@@ -66,63 +73,156 @@ class NeatRulesStore:
             return calculated_hash[:8]
         return calculated_hash
 
-    def import_rules(
-        self, importer: BaseImporter, validate: bool = True, client: NeatClient | None = None
-    ) -> IssueList:
-        if self.empty:
-            return self._import_rules(importer, validate, client)
+    def _rules_import_verify_convert(
+        self,
+        importer: BaseImporter,
+        validate: bool,
+        client: NeatClient | None = None,
+    ) -> tuple[InformationRules, DMSRules | None]:
+        """Action that imports rules, verifies them and optionally converts them."""
+        read_rules = importer.to_rules()
+        verified = VerifyAnyRules(validate, client).transform(read_rules)  # type: ignore[arg-type]
+        if isinstance(verified, InformationRules):
+            return verified, None
+        elif isinstance(verified, DMSRules):
+            return DMSToInformation().transform(verified), verified
         else:
-            # Importing can be used as a manual transformation.
-            return self._manual_transform(importer, validate, client)
+            # Bug in the code
+            raise ValueError(f"Invalid output from importer: {type(verified)}")
 
-    def _import_rules(
-        self, importer: BaseImporter, validate: bool = True, client: NeatClient | None = None
-    ) -> IssueList:
-        def action() -> tuple[InformationRules, DMSRules | None]:
-            read_rules = importer.to_rules()
-            verified = VerifyAnyRules(validate, client).transform(read_rules)  # type: ignore[arg-type]
-            if isinstance(verified, InformationRules):
-                return verified, None
-            elif isinstance(verified, DMSRules):
-                return DMSToInformation().transform(verified), verified
-            else:
-                # Bug in the code
-                raise ValueError(f"Invalid output from importer: {type(verified)}")
-
-        return self.import_action(action, importer)
+    def _graph_import_verify_convert(
+        self,
+        extractor: KnowledgeGraphExtractor,
+    ) -> tuple[InformationRules, DMSRules | None]:
+        info = extractor.get_information_rules()
+        dms: DMSRules | None = None
+        if isinstance(extractor, DMSGraphExtractor):
+            dms = extractor.get_dms_rules()
+        return info, dms
 
     def _manual_transform(
         self, importer: BaseImporter, validate: bool = True, client: NeatClient | None = None
     ) -> IssueList:
-        raise NotImplementedError("Manual transformation is not yet implemented.")
+        result, issue_list, start, end = self._do_activity(
+            partial(self._rules_import_verify_convert, importer, validate, client)
+        )
 
-    def import_graph(self, extractor: KnowledgeGraphExtractor) -> IssueList:
-        def action() -> tuple[InformationRules, DMSRules | None]:
-            info = extractor.get_information_rules()
-            dms: DMSRules | None = None
-            if isinstance(extractor, DMSGraphExtractor):
-                dms = extractor.get_dms_rules()
-            return info, dms
+        if not result:
+            return issue_list
+
+        info, dms = result
+        last_change = self.provenance[-1]
+
+        outside_agent = EXTERNAL_AGENT
+        outside_activity = Activity(
+            was_associated_with=outside_agent,
+            started_at_time=last_change.activity.ended_at_time,
+            ended_at_time=end,
+            used=last_change.target_entity,
+        )
+
+        # Case 1: Source of imported rules is not known
+        if not (source_id := self._get_source_id(result)):
+            raise NeatValueError(
+                "The source of the imported rules is unknown."
+                " Import will be skipped. Start a new NEAT session and import the data model there."
+            )
+
+        # Case 2: Source of imported rules not in rules_store
+        if not (source_entity := self.provenance.target_entity(source_id)) or not isinstance(
+            source_entity, RulesEntity
+        ):
+            raise NeatValueError(
+                "The source of the imported rules is not in the provenance."
+                " Import will be skipped. Start a new NEAT session and import the data model there."
+            )
+
+        # Case 3: Source is not the latest source entity in the provenance change
+        if source_entity.id_ != last_change.target_entity.id_:
+            raise NeatValueError(
+                "Imported rules are detached from the provenance chain."
+                " Import will be skipped. Start a new NEAT session and import the data model there."
+            )
+
+        # Case 4: Provenance is already at the physical state of the data model, going back to logical not possible
+        if not dms and source_entity.dms:
+            raise NeatValueError(
+                "Rules are already in physical state, import of logical model not possible."
+                " Import will be skipped. Start a new NEAT session and import the data model there."
+            )
+
+        # modification took place on information rules
+        if not dms and not source_entity.dms:
+            outside_target_entity = RulesEntity(
+                was_attributed_to=outside_agent,
+                was_generated_by=outside_activity,
+                information=info,
+                dms=dms,
+                issues=issue_list,
+                id_=self._create_id(info, dms),
+            )
+
+        # modification took place on dms rules, keep latest information rules
+        elif dms and source_entity.dms:
+            outside_target_entity = RulesEntity(
+                was_attributed_to=outside_agent,
+                was_generated_by=outside_activity,
+                information=last_change.target_entity.information,
+                dms=dms,
+                issues=issue_list,
+                id_=self._create_id(info, dms),
+            )
+
+        else:
+            raise NeatValueError("Invalid state of rules for manual transformation")
+
+        outside_change = Change(
+            source_entity=last_change.target_entity,
+            agent=outside_agent,
+            activity=outside_activity,
+            target_entity=outside_target_entity,
+            description="Manual transformation of rules outside of NEAT",
+        )
+
+        self._last_issues = issue_list
+        # record change that took place outside of neat
+        self.provenance.append(outside_change)
 
-        return self.import_action(action, extractor)
+        return issue_list
 
-    def import_action(
+    def import_graph(self, extractor: KnowledgeGraphExtractor) -> IssueList:
+        if not self.empty:
+            raise NeatValueError(f"Data model already exists. Cannot import {extractor.source_uri}.")
+        else:
+            return self.do_activity(partial(self._graph_import_verify_convert, extractor), extractor)
+
+    def import_rules(
         self,
-        action: Callable[[], tuple[InformationRules, DMSRules | None]],
-        agent_tool: BaseImporter | KnowledgeGraphExtractor,
+        importer: BaseImporter,
+        validate: bool = True,
+        client: NeatClient | None = None,
+        enable_manual_edit: bool = False,
     ) -> IssueList:
-        if self.provenance:
-            raise NeatValueError(f"Data model already exists. Cannot import {agent_tool.source_uri}.")
-        return self.do_activity(action, agent_tool)
+        if self.empty:
+            return self.do_activity(
+                partial(self._rules_import_verify_convert, importer, validate, client),
+                importer,
+            )
+        elif enable_manual_edit:
+            return self._manual_transform(importer, validate, client)
+        else:
+            raise NeatValueError("Re-importing rules in the rules store is not allowed.")
 
     def transform(self, *transformer: VerifiedRulesTransformer) -> IssueList:
         if not self.provenance:
             raise EmptyStore()
 
         all_issues = IssueList()
-        for item in transformer:
+        for agent_tool in transformer:
 
-            def action(transformer_item=item) -> tuple[InformationRules, DMSRules | None]:
+            def action(
+                transformer_item=agent_tool,
+            ) -> tuple[InformationRules, DMSRules | None]:
                 last_change = self.provenance[-1]
                 source_entity = last_change.target_entity
                 transformer_input = self._get_transformer_input(source_entity, transformer_item)
@@ -131,8 +231,9 @@ class NeatRulesStore:
                     return transformer_output, None
                 return last_change.target_entity.information, transformer_output
 
-            issues = self.do_activity(action, item)
+            issues = self.do_activity(action, agent_tool)
             all_issues.extend(issues)
+
         return all_issues
 
     def export(self, exporter: BaseExporter[T_VerifiedRules, T_Export]) -> T_Export:
@@ -154,34 +255,40 @@ class NeatRulesStore:
         self,
         action: Callable[[], tuple[InformationRules, DMSRules | None]],
         agent_tool: BaseImporter | VerifiedRulesTransformer | KnowledgeGraphExtractor,
-    ) -> IssueList:
+    ):
+        result, issue_list, start, end = self._do_activity(action)
+        self._last_issues = issue_list
+
+        if result:
+            self._update_provenance(agent_tool, result, issue_list, start, end)
+        return issue_list
+
+    def _update_provenance(
+        self,
+        agent_tool: BaseImporter | VerifiedRulesTransformer | KnowledgeGraphExtractor,
+        result: tuple[InformationRules, DMSRules | None],
+        issue_list: IssueList,
+        activity_start: datetime,
+        activity_end: datetime,
+    ) -> None:
+        # set source entity
         if isinstance(agent_tool, BaseImporter | KnowledgeGraphExtractor):
             source_entity = Entity.create_with_defaults(
                 was_attributed_to=UNKNOWN_AGENT,
                 id_=agent_tool.source_uri,
             )
         else:
-            # This is a transformer
             source_entity = self.provenance[-1].target_entity
 
-        start = datetime.now(timezone.utc)
-        result: tuple[InformationRules, DMSRules | None] | None = None
-        with catch_issues() as issue_list:
-            result = action()
-
-        end = datetime.now(timezone.utc)
-        self._last_issues = issue_list
-
+        # setting the rest of provenance components
+        info, dms = result
         agent = agent_tool.agent
         activity = Activity(
             was_associated_with=agent,
-            ended_at_time=end,
-            started_at_time=start,
+            ended_at_time=activity_end,
+            started_at_time=activity_start,
             used=source_entity,
         )
-        if result is None:
-            return issue_list
-        info, dms = result
 
         target_entity = RulesEntity(
             was_attributed_to=agent,
@@ -199,8 +306,20 @@ class NeatRulesStore:
             description=agent_tool.description,
             source_entity=source_entity,
         )
+
         self.provenance.append(change)
-        return issue_list
+
+    def _do_activity(
+        self,
+        action: Callable[[], tuple[InformationRules, DMSRules | None]],
+    ):
+        """This private method is used to execute an activity and return the result and issues."""
+        start = datetime.now(timezone.utc)
+        result: tuple[InformationRules, DMSRules | None] | None = None
+        with catch_issues() as issue_list:
+            result = action()
+        end = datetime.now(timezone.utc)
+        return result, issue_list, start, end
 
     def _export_activity(self, action: Callable, exporter: BaseExporter, target_id: URIRef, *exporter_args: Any) -> Any:
         if self.empty:
@@ -210,15 +329,18 @@ class NeatRulesStore:
         expected_types = exporter.source_types()
 
         if source_entity.dms is not None and isinstance(source_entity.dms, expected_types):
-            input_ = source_entity.dms
+            input_ = cast(VerifiedRules, source_entity.dms).model_copy(deep=True)
         elif isinstance(source_entity.information, expected_types):
-            input_ = source_entity.information
+            input_ = cast(VerifiedRules, source_entity.information).model_copy(deep=True)
         else:
             available: list[type] = [InformationRules]
             if source_entity.dms is not None:
                 available.append(DMSRules)
             raise InvalidActivityInput(expected=expected_types, have=tuple(available))
 
+        # need to write source prior the export
+        input_.metadata.source_id = source_entity.id_
+
         agent = exporter.agent
         start = datetime.now(timezone.utc)
         with catch_issues() as issue_list:
@@ -252,6 +374,7 @@ class NeatRulesStore:
             description=exporter.description,
             source_entity=source_entity,
         )
+
         self.exports_by_source_entity_id[source_entity.id_].append(change)
         if isinstance(result, UploadResultList):
             self._last_outcome = result
@@ -272,72 +395,14 @@ class NeatRulesStore:
         # Case 3: We have both information and dms rules and the transformer is compatible with information rules
         raise InvalidActivityInput(expected=(InformationRules,), have=(DMSRules,))
 
-    def _update_source_entity(self, source_entity: Entity, result: Rules, issue_list: IssueList) -> Entity:
-        """Update source entity to keep the unbroken provenance chain of changes."""
-
-        # Case 1: Store not empty, source of imported rules is not known
-        if not (source_id := self._get_source_id(result)):
-            raise NeatValueError(
-                "The data model to be read to the current NEAT session"
-                " has no relation to the session content."
-                " Import will be skipped."
-                "\n\nSuggestions:\n\t(1) Start a new NEAT session and "
-                "import the data model there."
-            )
-
-        # We are taking target entity as it is the entity that produce rules
-        # which were updated by activities outside of the rules tore
-        update_source_entity: Entity | None = self.provenance.target_entity(source_id)
-
-        # Case 2: source of imported rules not in rules_store
-        if not update_source_entity:
-            raise NeatValueError(
-                "The source of the data model being imported is not in"
-                " the content of this NEAT session."
-                " Import will be skipped."
-                "\n\nSuggestions:"
-                "\n\t(1) Start a new NEAT session and import the data model source"
-                "\n\t(2) Then import the data model itself"
-            )
+    def _get_source_id(self, result: tuple[InformationRules, DMSRules | None]) -> rdflib.URIRef | None:
+        """Return the source of the result.
 
-        # Case 3: source_entity in rules_store and but it is not the latest entity
-        if self.provenance[-1].target_entity.id_ != update_source_entity.id_:
-            raise NeatValueError(
-                "Source of imported data model is not the latest entity in the data model provenance."
-                "Pruning required to set the source entity to the latest entity in the provenance."
-            )
-
-        # Case 4: source_entity in rules_store and it is not the latest target entity
-        if self.provenance[-1].target_entity.id_ != update_source_entity.id_:
-            raise NeatValueError(
-                "Source of imported rules is not the latest entity in the provenance."
-                "Pruning required to set the source entity to the latest entity in the provenance."
-            )
-
-        # Case 5: source_entity in rules_store and it is the latest entity
-        # Here we need to check if the source and target entities are identical
-        # if they are ... we should raise an error and skip importing
-        # for now we will just return the source entity that we managed to extract
-
-        return update_source_entity or source_entity
-
-    def _get_source_id(self, result: Rules) -> rdflib.URIRef | None:
-        """Return the source of the result."""
-
-        if isinstance(result, ReadRules) and result.rules is not None and result.rules.metadata.source_id:
-            return rdflib.URIRef(result.rules.metadata.source_id)
-        if isinstance(result, VerifiedRules):
-            return result.metadata.source_id
-        return None
-
-    def _get_data_model_id(self, result: Rules) -> dm.DataModelId | None:
-        """Return the source of the result."""
-
-        if isinstance(result, ReadRules) and result.rules is not None:
-            return result.rules.metadata.as_data_model_id()
-        if isinstance(result, VerifiedRules):
-            return result.metadata.as_data_model_id()
-        return None
+        !!! note
+            This method prioritizes the source_id of the DMS rules
+        """
+        info, dms = result
+        return dms.metadata.source_id if dms else info.metadata.source_id
 
     def _create_id(self, info: InformationRules, dms: DMSRules | None) -> rdflib.URIRef:
         if dms is None:

cognite/neat/_utils/spreadsheet.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass, field
-from typing import Literal, cast, overload
+from typing import Any, Literal, cast, overload
 
 import pandas as pd
 from openpyxl import load_workbook
@@ -81,3 +81,12 @@ def _get_row_number(sheet: Worksheet, values_to_find: list[str]) -> int | None:
         if any(value in row for value in values_to_find):
             return row_number
     return None
+
+
+def find_column_with_value(sheet: Worksheet, value: Any) -> str | None:
+    for row in sheet.iter_rows():
+        for cell in row:
+            if cell.value and isinstance(cell.value, str) and cell.value.lower() == value.lower():
+                return cell.column_letter  # type: ignore
+
+    return None

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.109.0"
+__version__ = "0.109.2"
 __engine__ = "^2.0.3"

{cognite_neat-0.109.0.dist-info → cognite_neat-0.109.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cognite-neat
-Version: 0.109.0
+Version: 0.109.2
 Summary: Knowledge graph transformation
 License: Apache-2.0
 Author: Nikola Vasiljevic

{cognite_neat-0.109.0.dist-info → cognite_neat-0.109.2.dist-info}/RECORD

@@ -1,4 +1,5 @@
 cognite/neat/__init__.py,sha256=AXLMIF5t8RmjOpSaIlfqT8ltbPc__Tb8nAVbc1pkE0Q,176
+cognite/neat/_alpha.py,sha256=v-PRdeEikqmm4lHxGAQUOkC7WsySf5X3RPIHQwr2j2o,423
 cognite/neat/_client/__init__.py,sha256=RQ7MwL8mwGqGHokRzsPqO3XStDzmI4-c12_gz1UPJ74,177
 cognite/neat/_client/_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/_client/_api/data_modeling_loaders.py,sha256=0BZ5_NeszBvs4DHuHr0cGXytmvkTxrQ8k7v6iaX_mn4,35932
@@ -83,7 +84,7 @@ cognite/neat/_rules/catalog/info-rules-imf.xlsx,sha256=hj3ej7F1RYYyhGjX0rjVSOe_Z
 cognite/neat/_rules/exporters/__init__.py,sha256=IYBa0DIYlx8cFItgYRw9W4FY_LmVEjuaqMz3JORZZX0,1204
 cognite/neat/_rules/exporters/_base.py,sha256=VkNMy8wsH-x4tAjS44cXgzzNH0CM2k_4RhkMwK50J7g,2284
 cognite/neat/_rules/exporters/_rules2dms.py,sha256=7I3a8ZPwkIBQAClQbMjJ2D2aIITY-OBVUD-8hirCmzM,19183
-cognite/neat/_rules/exporters/_rules2excel.py,sha256=Y7QXtRoZNXuxM0gZIvy4u5_-V3r0qZ1ZfmMr-aMgH8A,12513
+cognite/neat/_rules/exporters/_rules2excel.py,sha256=z2KOL-sNqnv0xi4Kjuo_2YAPMDeGTdMxaBswcs41lPU,13756
 cognite/neat/_rules/exporters/_rules2instance_template.py,sha256=xICzDU8Ij3Q7FhYDbEjttLc8vwqbMajfN88lrTDpn9U,5944
 cognite/neat/_rules/exporters/_rules2ontology.py,sha256=l0QJKh0qJ5CDtReWDeQ2jt7wa__v0FEgVMX9jCKHT2U,22120
 cognite/neat/_rules/exporters/_rules2yaml.py,sha256=ggaPR8FO8PwZk1_nhwb5wVHk_C4s6qh1RrlbPkNcbBo,3160
@@ -121,7 +122,7 @@ cognite/neat/_rules/models/entities/_loaders.py,sha256=jFllRty5XpS6uLklr9wJkx7Bz
 cognite/neat/_rules/models/entities/_multi_value.py,sha256=6j-nlUA392yQP_uB_CFB6_qFReNhi54ZlbFTcOKpbKY,2755
 cognite/neat/_rules/models/entities/_single_value.py,sha256=GW1R8ko1vwjGhAs2Fl5BGgpz3qtiYnYoahT4hBgJdkA,19441
 cognite/neat/_rules/models/entities/_types.py,sha256=df9rnXJJKciv2Bp-Ve2q4xdEJt6WWniq12Z0hW2d6sk,1917
-cognite/neat/_rules/models/entities/_wrapped.py,sha256=-LP20mEv8H2NAoZplP-IJud-kaMHNFWE8fqbeJmdpk4,7579
+cognite/neat/_rules/models/entities/_wrapped.py,sha256=SpXIpglXBgaZt6geRparCWdY2F6SH0Fy99fcnA8x8Uk,7648
 cognite/neat/_rules/models/information/__init__.py,sha256=lV7l8RTfWBvN_DFkzea8OzADjq0rZGgWe_0Fiwtfje0,556
 cognite/neat/_rules/models/information/_rules.py,sha256=RQDcXEVZz9GG6KghpK-Vp4W_4ThoC12kfsPQdMkW75o,13766
 cognite/neat/_rules/models/information/_rules_input.py,sha256=njpi37IxTBLH4k51tRJeASU6yQXt324NSuxcZ_CjyU8,5940
@@ -136,29 +137,29 @@ cognite/neat/_rules/transformers/_mapping.py,sha256=lf-RKN__5Bg3-tZjEOCa1Sf_JtM_
 cognite/neat/_rules/transformers/_verification.py,sha256=jKTppklUCVwVlRfYyMfnUtV8r2ACTY-AtsoMF6L-KXo,4593
 cognite/neat/_session/__init__.py,sha256=fxQ5URVlUnmEGYyB8Baw7IDq-uYacqkigbc4b-Pr9Fw,58
 cognite/neat/_session/_base.py,sha256=WEAvz1b9Q8LsPOozLEUEOWLh9XWn1EtxwHw774qLi3E,11744
-cognite/neat/_session/_collector.py,sha256=SPCb1fEuOVIMHMQsVUNS7nkUUPhtUuNatnWPAIfQMcE,4093
+cognite/neat/_session/_collector.py,sha256=RcOGY0DjTCCKJt9j_p0gnQXn4omhsIX2G8Aq3ZFHIt4,4218
 cognite/neat/_session/_create.py,sha256=NATzf_5u2aYKXiBmDfNCrcdzQjQsE71XPi--f_iNGcE,5239
 cognite/neat/_session/_drop.py,sha256=gOkDAnddASpFxYxkPjlTyhkpNfnmDEj94GRI8tnHFR0,4167
 cognite/neat/_session/_fix.py,sha256=gpmbJ4TbB_v2nw4fEA2Qtf0ifO3UDEMHGdztha28S_U,898
-cognite/neat/_session/_inspect.py,sha256=-LdeyjmcVZbhuD7tWXaUfEiRpDHJUkpvIRiT-BzqhZ4,9025
+cognite/neat/_session/_inspect.py,sha256=VkmDfIjh49NPS8wNa1OdWqhW-oJy1EvMDhKiooMvcjI,9040
 cognite/neat/_session/_mapping.py,sha256=AkQwmqYH-0EgqoXHqCFwJY92hNSGzfojOelhVFlqH4c,2655
 cognite/neat/_session/_prepare.py,sha256=WoVp2Go2RnVaL2RvnCE2-X1kS4ZWxvupOlOmWTce0i0,11626
-cognite/neat/_session/_read.py,sha256=tx-sM8tdCFjwP8h_qXDlwYpfThuvu7s73vzM2tczqho,21941
+cognite/neat/_session/_read.py,sha256=V2F7InkvIODdML7jy4aGXH4PE91tWFzIGcNzOWnsTvM,22514
 cognite/neat/_session/_set.py,sha256=ZYR5G97fi-y68edbG-ftLqM_FRtn9G8V5zDtpslR9go,4070
 cognite/neat/_session/_show.py,sha256=3hUMWBlHCyMiVTbX1GmpVFfthpv3yBIZ293aU0mDxZ8,15156
-cognite/neat/_session/_state.py,sha256=KYmt1QEFu0LoXBuheu0ARA9gq2FsQvjyDWcJt_OiPHQ,3939
-cognite/neat/_session/_to.py,sha256=e6A2BeSL9NjQz1Fk2_RaJom7NPZErILqGi_UFDXWCZQ,10897
+cognite/neat/_session/_state.py,sha256=gDyt2gf2kuiHu42zDMO-Pg0zjRllEKC4O8lJ0pMGCkU,4059
+cognite/neat/_session/_to.py,sha256=w7ukhG0cjkKlMPhhivaL5Poq4W_-p8ZEWQSWuVnQGiw,11670
 cognite/neat/_session/_wizard.py,sha256=9idlzhZy54h2Iwupe9iXKX3RDb5jJQuBZFEouni50L0,1476
 cognite/neat/_session/engine/__init__.py,sha256=D3MxUorEs6-NtgoICqtZ8PISQrjrr4dvca6n48bu_bI,120
 cognite/neat/_session/engine/_import.py,sha256=1QxA2_EK613lXYAHKQbZyw2yjo5P9XuiX4Z6_6-WMNQ,169
 cognite/neat/_session/engine/_interface.py,sha256=ItJ1VMrPt-pKKvpSpglD9n9yFC6ehF9xV2DUVCyfQB0,533
 cognite/neat/_session/engine/_load.py,sha256=LcoYVthQyCzLWKzRE_75_nElS-n_eMWSPAgNJBnh5dA,5193
-cognite/neat/_session/exceptions.py,sha256=6wx3gAqU_nuXMAwuPijGvJg3NSSimXyNSefWa_p5Lo8,2717
+cognite/neat/_session/exceptions.py,sha256=XCLVaEe7SMIyNW7mwt0WF7_e5K8KfV9yHkEtsjL60Tg,3182
 cognite/neat/_shared.py,sha256=Ov59SWYboRRsncB_5V1ZC_BAoACfNLjo80vqE5Ru6wo,2325
 cognite/neat/_store/__init__.py,sha256=RrvuZrMdezqun5dOrwHWSk26kampZcvqiHBqSFumkEE,130
 cognite/neat/_store/_graph_store.py,sha256=ru0wuJtBAZoN2HJOJhBmldAsbRpp9zMcIcwfTzEcsN0,26882
-cognite/neat/_store/_provenance.py,sha256=TgEtFc3RmdYO_hAj7NWgB_sj57DGoSxOiIFRAdYVeNM,7168
-cognite/neat/_store/_rules_store.py,sha256=ABBdEElx3va4F62IFebF99tkGF1cQ3qfQ0ibVnGHRtQ,16218
+cognite/neat/_store/_provenance.py,sha256=0HeWyzvfCTxQiG2PygLLqWpdCAooVsFRckpGsnJbwzk,7260
+cognite/neat/_store/_rules_store.py,sha256=R-bkiNiwqGQ9udrEmamvuEidQsemFpGBif1zmf4H-QA,17713
 cognite/neat/_store/exceptions.py,sha256=fbed_CGDdYU4oELgCL0_c5d85HGrUiYvXmL2D0WIDww,1593
 cognite/neat/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/_utils/auth.py,sha256=hpNQjXpM3i7r0RU13ZLen1sa78nvPd4E1Sh3j1TMK4g,14701
@@ -169,15 +170,15 @@ cognite/neat/_utils/io_.py,sha256=D2Mg8sOxfBoDg3fC0jBzaxO3vkXmr0QvZSgYIv6xRkM,38
 cognite/neat/_utils/rdf_.py,sha256=b3sE3aTW9lu4gJWQJEaq_NCLbI54cc1o12utz-xbLh8,9023
 cognite/neat/_utils/reader/__init__.py,sha256=fPkrNB_9hLB7CyHTCFV_xEbIfOMqUQzNly5JN33-QfM,146
 cognite/neat/_utils/reader/_base.py,sha256=Q35hz8tqAiQiELjE4DsDDKQHLtRmSTrty4Gep9rg_CU,5444
-cognite/neat/_utils/spreadsheet.py,sha256=ssNWat42srjbQfzzApaGiNanyEy-oFlAQ6qQXCZOhoE,2777
+cognite/neat/_utils/spreadsheet.py,sha256=20L_44m0hg3UdBFwCxnmAravxvCOolNZwVOGHfjYGR4,3089
 cognite/neat/_utils/text.py,sha256=0IffvBIAmeGh92F4T6xiEdd-vv3B7FOGEMbfuTktO5Y,4017
 cognite/neat/_utils/time_.py,sha256=O30LUiDH9TdOYz8_a9pFqTtJdg8vEjC3qHCk8xZblG8,345
 cognite/neat/_utils/upload.py,sha256=iWKmsQgw4EHLv-11NjYu7zAj5LtqTAfNa87a1kWeuaU,5727
 cognite/neat/_utils/xml_.py,sha256=FQkq84u35MUsnKcL6nTMJ9ajtG9D5i1u4VBnhGqP2DQ,1710
-cognite/neat/_version.py,sha256=c4wxycTdwtXNOhkXzQOSDR2zGhasMvyx0oPIJFa4JQ0,46
+cognite/neat/_version.py,sha256=C9cGHhgqgxsB1rbURMXLvpzrANgFHbmOFq3MJSffn3w,46
 cognite/neat/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cognite_neat-0.109.0.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-0.109.0.dist-info/METADATA,sha256=K9HPjBYFzqFnNVscJ5F8iD91b0p-ovHgijdYwflgGNc,5361
-cognite_neat-0.109.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-cognite_neat-0.109.0.dist-info/entry_points.txt,sha256=SsQlnl8SNMSSjE3acBI835JYFtsIinLSbVmHmMEXv6E,51
-cognite_neat-0.109.0.dist-info/RECORD,,
+cognite_neat-0.109.2.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-0.109.2.dist-info/METADATA,sha256=lskGgPFezWb9blta8e5kdLXvTROMUKyMpVxrh8yIQvI,5361
+cognite_neat-0.109.2.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+cognite_neat-0.109.2.dist-info/entry_points.txt,sha256=SsQlnl8SNMSSjE3acBI835JYFtsIinLSbVmHmMEXv6E,51
+cognite_neat-0.109.2.dist-info/RECORD,,