cognite-neat 0.103.0__py3-none-any.whl → 0.104.0__py3-none-any.whl

cognite/neat/_store/_rules_store.py

@@ -1,20 +1,382 @@
-from cognite.neat._issues import IssueList
+import hashlib
+from collections import defaultdict
+from collections.abc import Callable, Hashable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from functools import partial
+from pathlib import Path
+from typing import Any, cast
+
+import rdflib
+
+from cognite.neat._client import NeatClient
+from cognite.neat._constants import DEFAULT_NAMESPACE
+from cognite.neat._issues import IssueList, catch_issues
+from cognite.neat._issues.errors import NeatValueError
+from cognite.neat._rules._shared import ReadRules, Rules, T_VerifiedRules, VerifiedRules
 from cognite.neat._rules.exporters import BaseExporter
+from cognite.neat._rules.exporters._base import CDFExporter, T_Export
 from cognite.neat._rules.importers import BaseImporter
+from cognite.neat._rules.models import DMSRules, InformationRules
+from cognite.neat._rules.models._base_input import InputRules
 from cognite.neat._rules.transformers import RulesTransformer
+from cognite.neat._utils.upload import UploadResultList
+
+from ._provenance import EMPTY_ENTITY, UNKNOWN_AGENT, Activity, Agent, Change, Entity, Provenance
+from .exceptions import EmptyStore, InvalidInputOperation
+
+
+@dataclass(frozen=True)
+class ModelEntity(Entity):
+    result: Rules | None = None
 
-from ._provenance import Provenance
+    @property
+    def display_name(self) -> str:
+        if self.result is None:
+            return "Failed"
+        if isinstance(self.result, ReadRules):
+            if self.result.rules is None:
+                return "FailedRead"
+            return self.result.rules.display_type_name()
+        else:
+            return self.result.display_type_name()
+
+
+@dataclass(frozen=True)
+class OutcomeEntity(Entity):
+    result: UploadResultList | Path | str | None = None
 
 
 class NeatRulesStore:
-    def __init__(self):
-        self._provenance = Provenance()
+    def __init__(self) -> None:
+        self.provenance = Provenance()
+        self.exports_by_source_entity_id: dict[rdflib.URIRef, list[Change]] = defaultdict(list)
+        self.pruned_by_source_entity_id: dict[rdflib.URIRef, list[Provenance]] = defaultdict(list)
+        self._last_outcome: UploadResultList | None = None
+        self._iteration_by_id: dict[Hashable, int] = {}
+
+    def calculate_provenance_hash(self, shorten: bool = True) -> str:
+        sha256_hash = hashlib.sha256()
+        for change in self.provenance:
+            for id_ in [change.agent.id_, change.activity.id_, change.target_entity.id_, change.source_entity.id_]:
+                sha256_hash.update(str(id_).encode("utf-8"))
+        calculated_hash = sha256_hash.hexdigest()
+        if shorten:
+            return calculated_hash[:8]
+        return calculated_hash
+
+    def import_(self, importer: BaseImporter) -> IssueList:
+        agent = importer.agent
+        source_entity = Entity(
+            was_attributed_to=UNKNOWN_AGENT,
+            id_=importer.source_uri,
+        )
+        return self._do_activity(importer.to_rules, agent, source_entity, importer.description)[1]
+
+    def transform(self, *transformer: RulesTransformer) -> IssueList:
+        if not self.provenance:
+            raise EmptyStore()
+
+        all_issues = IssueList()
+        for item in transformer:
+            last_change = self.provenance[-1]
+            source_entity = last_change.target_entity
+            if not isinstance(source_entity, ModelEntity):
+                # Todo: Provenance should be of an entity type
+                raise ValueError("Bug in neat: The last entity in the provenance is not a model entity.")
+            transformer_input = source_entity.result
+
+            if not item.is_valid_input(transformer_input):
+                raise InvalidInputOperation(expected=item.transform_type_hint(), got=type(transformer_input))
+
+            transform_issues = self._do_activity(
+                partial(item.transform, rules=transformer_input),
+                item.agent,
+                source_entity,
+                item.description,
+            )[1]
+            all_issues.extend(transform_issues)
+        return all_issues
+
+    def export(self, exporter: BaseExporter[T_VerifiedRules, T_Export]) -> T_Export:
+        last_change = self.provenance[-1]
+        source_entity = last_change.target_entity
+        if not isinstance(source_entity, ModelEntity):
+            # Todo: Provenance should be of an entity type
+            raise ValueError("Bug in neat: The last entity in the provenance is not a model entity.")
+        expected_types = exporter.source_types()
+        if not any(isinstance(source_entity.result, type_) for type_ in expected_types):
+            raise InvalidInputOperation(expected=expected_types, got=type(source_entity.result))
+
+        agent = exporter.agent
+        start = datetime.now(timezone.utc)
+        issue_list = IssueList()
+        with catch_issues(issue_list) as _:
+            # Validate the type of the result
+            result = exporter.export(source_entity.result)  # type: ignore[arg-type]
+        end = datetime.now(timezone.utc)
+        target_id = DEFAULT_NAMESPACE["export-result"]
+        activity = Activity(
+            was_associated_with=agent,
+            ended_at_time=end,
+            started_at_time=start,
+            used=source_entity,
+        )
+        target_entity = OutcomeEntity(
+            was_attributed_to=agent,
+            was_generated_by=activity,
+            result=type(result).__name__,
+            issues=issue_list,
+            id_=target_id,
+        )
+        change = Change(
+            agent=agent,
+            activity=activity,
+            target_entity=target_entity,
+            description=exporter.description,
+            source_entity=source_entity,
+        )
+        self.exports_by_source_entity_id[source_entity.id_].append(change)
+        return result
+
+    def export_to_file(self, exporter: BaseExporter, path: Path) -> None:
+        last_change = self.provenance[-1]
+        source_entity = last_change.target_entity
+        if not isinstance(source_entity, ModelEntity):
+            # Todo: Provenance should be of an entity type
+            raise ValueError("Bug in neat: The last entity in the provenance is not a model entity.")
+        expected_types = exporter.source_types()
+        if not any(isinstance(source_entity.result, type_) for type_ in expected_types):
+            raise InvalidInputOperation(expected=expected_types, got=type(source_entity.result))
+        target_id = DEFAULT_NAMESPACE[path.name]
+        agent = exporter.agent
+        start = datetime.now(timezone.utc)
+        issue_list = IssueList()
+        with catch_issues(issue_list) as _:
+            exporter.export_to_file(source_entity.result, path)
+        end = datetime.now(timezone.utc)
+
+        activity = Activity(
+            was_associated_with=agent,
+            ended_at_time=end,
+            started_at_time=start,
+            used=source_entity,
+        )
+        target_entity = OutcomeEntity(
+            was_attributed_to=agent,
+            was_generated_by=activity,
+            result=path,
+            issues=issue_list,
+            id_=target_id,
+        )
+        change = Change(
+            agent=agent,
+            activity=activity,
+            target_entity=target_entity,
+            description=exporter.description,
+            source_entity=source_entity,
+        )
+        self.exports_by_source_entity_id[source_entity.id_].append(change)
+
+    def export_to_cdf(self, exporter: CDFExporter, client: NeatClient, dry_run: bool) -> UploadResultList:
+        last_change = self.provenance[-1]
+        source_entity = last_change.target_entity
+        if not isinstance(source_entity, ModelEntity):
+            # Todo: Provenance should be of an entity type
+            raise ValueError("Bug in neat: The last entity in the provenance is not a model entity.")
+        expected_types = exporter.source_types()
+        if not any(isinstance(source_entity.result, type_) for type_ in expected_types):
+            raise InvalidInputOperation(expected=expected_types, got=type(source_entity.result))
+
+        agent = exporter.agent
+        start = datetime.now(timezone.utc)
+        target_id = DEFAULT_NAMESPACE["upload-result"]
+        issue_list = IssueList()
+        result: UploadResultList | None = None
+        with catch_issues(issue_list) as _:
+            result = exporter.export_to_cdf(source_entity.result, client, dry_run)
+        end = datetime.now(timezone.utc)
+
+        activity = Activity(
+            was_associated_with=agent,
+            ended_at_time=end,
+            started_at_time=start,
+            used=source_entity,
+        )
+        target_entity = OutcomeEntity(
+            was_attributed_to=agent,
+            was_generated_by=activity,
+            result=result,
+            issues=issue_list,
+            id_=target_id,
+        )
+        change = Change(
+            agent=agent,
+            activity=activity,
+            target_entity=target_entity,
+            description=exporter.description,
+            source_entity=source_entity,
+        )
+        self.exports_by_source_entity_id[source_entity.id_].append(change)
+        self._last_outcome = result
+        return result
+
+    def prune_until_compatible(self, transformer: RulesTransformer) -> list[Change]:
+        """Prune the provenance until the last successful entity is compatible with the transformer.
+
+        Args:
+            transformer: The transformer to check compatibility with.
+
+        Returns:
+            The changes that were pruned.
+        """
+        pruned_candidates: list[Change] = []
+        for change in reversed(self.provenance):
+            if not isinstance(change.target_entity, ModelEntity):
+                continue
+            if not transformer.is_valid_input(change.target_entity.result):
+                pruned_candidates.append(change)
+            else:
+                break
+        else:
+            raise NeatValueError("No compatible entity found in the provenance.")
+        if not pruned_candidates:
+            return []
+        self.provenance = self.provenance[: -len(pruned_candidates)]
+        pruned_candidates.reverse()
+        self.pruned_by_source_entity_id[self.provenance[-1].target_entity.id_].append(Provenance(pruned_candidates))
+        return pruned_candidates
+
+    def _export(self, action: Callable[[Any], Any], agent: Agent, description: str) -> Any:
+        last_entity: ModelEntity | None = None
+        for change in reversed(self.provenance):
+            if isinstance(change.target_entity, ModelEntity) and isinstance(change.target_entity.result, DMSRules):
+                last_entity = change.target_entity
+                break
+        if last_entity is None:
+            raise NeatValueError("No verified DMS rules found in the provenance.")
+        rules = last_entity.result
+        result, _ = self._do_activity(lambda: action(rules), agent, last_entity, description)
+        return result
+
+    def _do_activity(
+        self, action: Callable[[], Rules | None], agent: Agent, source_entity: Entity, description: str
+    ) -> tuple[Any, IssueList]:
+        start = datetime.now(timezone.utc)
+        issue_list = IssueList()
+        result: Rules | None = None
+        with catch_issues(issue_list) as _:
+            result = action()
+        end = datetime.now(timezone.utc)
+
+        activity = Activity(
+            was_associated_with=agent,
+            ended_at_time=end,
+            started_at_time=start,
+            used=source_entity,
+        )
+        target_entity = ModelEntity(
+            was_attributed_to=agent,
+            was_generated_by=activity,
+            result=result,
+            issues=issue_list,
+            id_=self._create_id(result),
+        )
+        change = Change(
+            agent=agent,
+            activity=activity,
+            target_entity=target_entity,
+            description=description,
+            source_entity=source_entity,
+        )
+
+        self.provenance.append(change)
+        return result, issue_list
+
+    def _create_id(self, result: Any) -> rdflib.URIRef:
+        identifier: rdflib.URIRef
+        if result is None:
+            identifier = EMPTY_ENTITY.id_
+        elif isinstance(result, ReadRules):
+            if result.rules is None:
+                identifier = EMPTY_ENTITY.id_
+            else:
+                identifier = result.rules.metadata.identifier
+        elif isinstance(result, VerifiedRules):
+            identifier = result.metadata.identifier
+        else:
+            identifier = DEFAULT_NAMESPACE["unknown-entity"]
+
+        if identifier not in self._iteration_by_id:
+            self._iteration_by_id[identifier] = 1
+            return identifier
+        self._iteration_by_id[identifier] += 1
+        return identifier + f"/Iteration_{self._iteration_by_id[identifier]}"
+
+    def get_last_entity(self) -> ModelEntity:
+        if not self.provenance:
+            raise NeatValueError("No entity found in the provenance.")
+        return cast(ModelEntity, self.provenance[-1].target_entity)
+
+    def get_last_successful_entity(self) -> ModelEntity:
+        for change in reversed(self.provenance):
+            if isinstance(change.target_entity, ModelEntity) and change.target_entity.result:
+                return change.target_entity
+        raise NeatValueError("No successful entity found in the provenance.")
+
+    @property
+    def has_unverified_rules(self) -> bool:
+        return any(
+            isinstance(change.target_entity, ModelEntity)
+            and isinstance(change.target_entity.result, ReadRules)
+            and change.target_entity.result.rules is not None
+            for change in self.provenance
+        )
+
+    @property
+    def has_verified_rules(self) -> bool:
+        return any(
+            isinstance(change.target_entity, ModelEntity)
+            and isinstance(change.target_entity.result, DMSRules | InformationRules)
+            for change in self.provenance
+        )
+
+    @property
+    def last_unverified_rule(self) -> InputRules:
+        for change in reversed(self.provenance):
+            if (
+                isinstance(change.target_entity, ModelEntity)
+                and isinstance(change.target_entity.result, ReadRules)
+                and change.target_entity.result.rules is not None
+            ):
+                return change.target_entity.result.rules
+
+        raise NeatValueError("No unverified rule found in the provenance.")
+
+    @property
+    def last_verified_rule(self) -> DMSRules | InformationRules:
+        for change in reversed(self.provenance):
+            if isinstance(change.target_entity, ModelEntity) and isinstance(
+                change.target_entity.result, DMSRules | InformationRules
+            ):
+                return change.target_entity.result
+        raise NeatValueError("No verified rule found in the provenance.")
 
-    def write(self, importer: BaseImporter) -> IssueList:
-        raise NotImplementedError()
+    @property
+    def last_verified_dms_rules(self) -> DMSRules:
+        for change in reversed(self.provenance):
+            if isinstance(change.target_entity, ModelEntity) and isinstance(change.target_entity.result, DMSRules):
+                return change.target_entity.result
+        raise NeatValueError("No verified DMS rules found in the provenance.")
 
-    def transform(self, transformer: RulesTransformer) -> IssueList:
-        raise NotImplementedError()
+    @property
+    def last_issues(self) -> IssueList:
+        if not self.provenance:
+            raise NeatValueError("No issues found in the provenance.")
+        return self.provenance[-1].target_entity.issues
 
-    def read(self, exporter: BaseExporter) -> IssueList:
-        raise NotImplementedError()
+    @property
+    def last_outcome(self) -> UploadResultList:
+        if self._last_outcome is not None:
+            return self._last_outcome
+        raise NeatValueError("No outcome found in the provenance.")

cognite/neat/_store/exceptions.py

@@ -0,0 +1,23 @@
+"""These are special exceptions that are used by the store to signal invalid transformers"""
+
+from dataclasses import dataclass
+
+
+class NeatStoreError(Exception):
+    """Base class for all exceptions in the store module"""
+
+    ...
+
+
+@dataclass
+class InvalidInputOperation(NeatStoreError, RuntimeError):
+    """Raised when an invalid operation is attempted"""
+
+    expected: tuple[type, ...]
+    got: type
+
+
+class EmptyStore(NeatStoreError, RuntimeError):
+    """Raised when the store is empty"""
+
+    ...

cognite/neat/_utils/graph_transformations_report.py

@@ -0,0 +1,36 @@
+from abc import ABC
+from dataclasses import dataclass
+from typing import Any
+
+from cognite.neat._shared import NeatList, NeatObject
+
+
+@dataclass
+class GraphTransformationResult(NeatObject, ABC):
+    name: str
+    affected_nodes_count: int | None = None
+    added: int | None = None
+    removed: int | None = None
+    skipped: int | None = None
+    modified: int | None = None
+
+    def dump(self, aggregate: bool = True) -> dict[str, Any]:
+        output: dict[str, Any] = {"name": self.name}
+        if self.added:
+            output["added"] = self.added
+        if self.removed:
+            output["removed"] = self.removed
+        if self.skipped:
+            output["skipped"] = self.skipped
+        if self.affected_nodes_count:
+            output["affected nodes"] = self.affected_nodes_count
+        if self.modified:
+            output["modified instances"] = self.modified
+        return output
+
+
+class GraphTransformationResultList(NeatList[GraphTransformationResult]):
+    def _repr_html_(self) -> str:
+        df = self.to_pandas().fillna(0)
+        df = df.style.format({column: "{:,.0f}".format for column in df.select_dtypes(include="number").columns})
+        return df._repr_html_()  # type: ignore[attr-defined]

cognite/neat/_utils/io_.py

@@ -0,0 +1,11 @@
+import re
+
+# Spaces are allowed, but we replace them as well
+_ILLEGAL_CHARACTERS = re.compile(r"[<>:\"/\\|?*\s]")
+
+
+def to_directory_compatible(text: str) -> str:
+    """Convert a string to be compatible with directory names on all platforms"""
+    cleaned = _ILLEGAL_CHARACTERS.sub("_", text)
+    # Replace multiple underscores with a single one
+    return re.sub(r"_+", "_", cleaned)

cognite/neat/_utils/rdf_.py

@@ -257,3 +257,11 @@ def remove_instance_ids_in_batch(graph: Graph, instance_ids: Iterable[URIRef], b
         check_commit()
 
     check_commit(force_commit=True)
+
+
+def uri_display_name(thing: URIRef) -> str:
+    if "https://cognitedata.com/dms/data-model/" in thing:
+        return "DMS(" + ",".join(thing.replace("https://cognitedata.com/dms/data-model/", "").split("/")) + ")"
+    elif "http://purl.org/cognite/neat/data-model/" in thing:
+        return "NEAT(" + ",".join(thing.replace("http://purl.org/cognite/neat/data-model/", "").split("/")) + ")"
+    return remove_namespace_from_uri(thing)

cognite/neat/_utils/spreadsheet.py

@@ -14,7 +14,7 @@ class SpreadsheetRead:
     such that the error/warning messages are accurate.
     """
 
-    header_row: int = 0
+    header_row: int = 1
     empty_rows: list[int] = field(default_factory=list)
     is_one_indexed: bool = True
 
@@ -22,13 +22,13 @@ class SpreadsheetRead:
         self.empty_rows = sorted(self.empty_rows)
 
     def adjusted_row_number(self, row_no: int) -> int:
-        output = row_no + self.header_row + (1 if self.is_one_indexed else 0)
+        output = row_no
         for empty_row in self.empty_rows:
             if empty_row <= output:
                 output += 1
             else:
                 break
-        return output
+        return output + self.header_row + (1 if self.is_one_indexed else 0)
 
 
 @overload
@@ -71,7 +71,8 @@ def read_individual_sheet(
         raw["Value Type"] = raw["Value Type"].replace(float("nan"), "#N/A")
     output = raw.replace(float("nan"), None).to_dict(orient="records")
     if return_read_info:
-        return output, SpreadsheetRead(header_row=skiprows, empty_rows=empty_rows, is_one_indexed=True)
+        # If no rows are skipped, row 1 is the header row.
+        return output, SpreadsheetRead(header_row=skiprows + 1, empty_rows=empty_rows, is_one_indexed=True)
     return output
 
 

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.103.0"
+__version__ = "0.104.0"
 __engine__ = "^2.0.3"

cognite/neat/_workflows/steps/lib/current/rules_exporter.py

@@ -89,7 +89,7 @@ class DeleteDataModelFromCDF(Step):
         if isinstance(input_rules, DMSRules):
             dms_rules = input_rules
         elif isinstance(input_rules, InformationRules):
-            dms_rules = InformationToDMS().transform(input_rules).rules
+            dms_rules = InformationToDMS().transform(input_rules)
         else:
             raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
 
@@ -201,7 +201,7 @@ class RulesToDMS(Step):
         if isinstance(input_rules, DMSRules):
             dms_rules = input_rules
         elif isinstance(input_rules, InformationRules):
-            dms_rules = InformationToDMS().transform(input_rules).rules
+            dms_rules = InformationToDMS().transform(input_rules)
         else:
             raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
 
@@ -324,12 +324,12 @@ class RulesToExcel(Step):
             ...
         elif output_role is RoleTypes.dms:
             if isinstance(rule_instance, InformationRules):
-                rule_instance = InformationToDMS().transform(rule_instance).rules
+                rule_instance = InformationToDMS().transform(rule_instance)
             else:
                 raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
         elif output_role is RoleTypes.information:
             if isinstance(rule_instance, DMSRules):
-                rule_instance = DMSToInformation().transform(rule_instance).rules
+                rule_instance = DMSToInformation().transform(rule_instance)
             else:
                 raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
         else:
@@ -394,7 +394,7 @@ class RulesToOntology(Step):
 
         input_rules = rules.information or rules.dms
         if isinstance(input_rules, DMSRules):
-            info_rules = DMSToInformation().transform(input_rules).rules
+            info_rules = DMSToInformation().transform(input_rules)
         elif isinstance(input_rules, InformationRules):
             info_rules = input_rules
         else:
@@ -453,7 +453,7 @@ class RulesToSHACL(Step):
 
         input_rules = rules.information or rules.dms
         if isinstance(input_rules, DMSRules):
-            info_rules = DMSToInformation().transform(input_rules).rules
+            info_rules = DMSToInformation().transform(input_rules)
         elif isinstance(input_rules, InformationRules):
             info_rules = input_rules
         else:
@@ -511,7 +511,7 @@ class RulesToSemanticDataModel(Step):
         storage_path.parent.mkdir(parents=True, exist_ok=True)
         input_rules = rules.information or rules.dms
         if isinstance(input_rules, DMSRules):
-            info_rules = DMSToInformation().transform(input_rules).rules
+            info_rules = DMSToInformation().transform(input_rules)
         elif isinstance(input_rules, InformationRules):
             info_rules = input_rules
         else: