cognite-neat 0.96.6__py3-none-any.whl → 0.97.0__py3-none-any.whl

cognite/neat/_session/_prepare.py

@@ -1,12 +1,15 @@
 from collections.abc import Collection
-from typing import Literal, cast
+from datetime import datetime, timezone
+from typing import Literal
 
 from cognite.client.data_classes.data_modeling import DataModelIdentifier
+from rdflib import URIRef
 
-from cognite.neat._issues._base import IssueList
+from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
 from cognite.neat._rules._shared import ReadRules
 from cognite.neat._rules.models.information._rules_input import InformationInputRules
 from cognite.neat._rules.transformers import ReduceCogniteModel, ToCompliantEntities, ToExtension
+from cognite.neat._store._provenance import Change
 
 from ._state import SessionState
 from .exceptions import intercept_session_exceptions
@@ -18,6 +21,52 @@ class PrepareAPI:
         self._state = state
         self._verbose = verbose
         self.data_model = DataModelPrepareAPI(state, verbose)
+        self.instances = InstancePrepareAPI(state, verbose)
+
+
+@intercept_session_exceptions
+class InstancePrepareAPI:
+    def __init__(self, state: SessionState, verbose: bool) -> None:
+        self._state = state
+        self._verbose = verbose
+
+    def make_connection_on_exact_match(
+        self,
+        source: tuple[URIRef, URIRef],
+        target: tuple[URIRef, URIRef],
+        connection: URIRef | None = None,
+        limit: int | None = 100,
+    ) -> None:
+        """Make connection on exact match.
+
+        Args:
+            source: The source of the connection. A tuple of (rdf type, property) where
+                    where property is the property that should be matched on the source
+                    to make the connection with the target.
+            target: The target of the connection. A tuple of (rdf type, property) where
+                    where property is the property that should be matched on the target
+                    to make the connection with the source.
+
+            connection: new property to use for the connection. If None, the connection
+                    will be made by lowercasing the target type.
+            limit: The maximum number of connections to make. If None, all connections
+
+
+        """
+
+        subject_type, subject_predicate = source
+        object_type, object_predicate = target
+
+        transformer = MakeConnectionOnExactMatch(
+            subject_type,
+            subject_predicate,
+            object_type,
+            object_predicate,
+            connection,
+            limit,
+        )
+
+        self._state.instances.store.transform(transformer)
 
 
 @intercept_session_exceptions
@@ -28,21 +77,32 @@ class DataModelPrepareAPI:
 
     def cdf_compliant_external_ids(self) -> None:
         """Convert data model component external ids to CDF compliant entities."""
-        if input := self._state.information_input_rule:
-            output = ToCompliantEntities().transform(input)
-            self._state.input_rules.append(
-                ReadRules(
-                    rules=cast(InformationInputRules, output.get_rules()),
-                    issues=IssueList(),
-                    read_context={},
-                )
+        if input := self._state.data_model.last_info_unverified_rule:
+            source_id, rules = input
+
+            start = datetime.now(timezone.utc)
+            transformer = ToCompliantEntities()
+            output: ReadRules[InformationInputRules] = transformer.transform(rules)
+            end = datetime.now(timezone.utc)
+
+            change = Change.from_rules_activity(
+                output,
+                transformer.agent,
+                start,
+                end,
+                "Converted external ids to CDF compliant entities",
+                self._state.data_model.provenance.source_entity(source_id)
+                or self._state.data_model.provenance.target_entity(source_id),
             )
 
+            self._state.data_model.write(output, change)
+
     def to_enterprise(
         self,
         data_model_id: DataModelIdentifier,
         org_name: str = "My",
         dummy_property: str = "GUID",
+        move_connections: bool = False,
     ) -> None:
         """Uses the current data model as a basis to create enterprise data model
 
@@ -50,6 +110,7 @@ class DataModelPrepareAPI:
             data_model_id: The enterprise data model id that is being created
             org_name: Organization name to use for the views in the enterprise data model.
             dummy_property: The dummy property to use as placeholder for the views in the new data model.
+            move_connections: If True, the connections will be moved to the new data model.
 
         !!! note "Enterprise Data Model Creation"
             Always create an enterprise data model from a Cognite Data Model as this will
@@ -58,15 +119,39 @@ class DataModelPrepareAPI:
             - Atlas AI
             - ...
 
+        !!! note "Move Connections"
+            If you want to move the connections to the new data model, set the move_connections
+            to True. This will move the connections to the new data model and use new model
+            views as the source and target views.
+
         """
-        if dms := self._state.last_verified_dms_rules:
-            output = ToExtension(
+        if input := self._state.data_model.last_verified_dms_rules:
+            source_id, rules = input
+
+            start = datetime.now(timezone.utc)
+            transformer = ToExtension(
                 new_model_id=data_model_id,
                 org_name=org_name,
                 type_="enterprise",
                 dummy_property=dummy_property,
-            ).transform(dms)
-            self._state.verified_rules.append(output.rules)
+                move_connections=move_connections,
+            )
+            output = transformer.transform(rules)
+            end = datetime.now(timezone.utc)
+
+            change = Change.from_rules_activity(
+                output,
+                transformer.agent,
+                start,
+                end,
+                (
+                    f"Prepared data model {data_model_id} to be enterprise data "
+                    f"model on top of {rules.metadata.as_data_model_id()}"
+                ),
+                self._state.data_model.provenance.source_entity(source_id),
+            )
+
+            self._state.data_model.write(output.rules, change)
 
     def to_solution(
         self,
@@ -94,15 +179,33 @@ class DataModelPrepareAPI:
             the containers in the solution data model space.
 
         """
-        if dms := self._state.last_verified_dms_rules:
-            output = ToExtension(
+        if input := self._state.data_model.last_verified_dms_rules:
+            source_id, rules = input
+
+            start = datetime.now(timezone.utc)
+            transformer = ToExtension(
                 new_model_id=data_model_id,
                 org_name=org_name,
                 type_="solution",
                 mode=mode,
                 dummy_property=dummy_property,
-            ).transform(dms)
-            self._state.verified_rules.append(output.rules)
+            )
+            output = transformer.transform(rules)
+            end = datetime.now(timezone.utc)
+
+            change = Change.from_rules_activity(
+                output,
+                transformer.agent,
+                start,
+                end,
+                (
+                    f"Prepared data model {data_model_id} to be solution data model "
+                    f"on top of {rules.metadata.as_data_model_id()}"
+                ),
+                self._state.data_model.provenance.source_entity(source_id),
+            )
+
+            self._state.data_model.write(output.rules, change)
 
     def reduce(self, drop: Collection[Literal["3D", "Annotation", "BaseViews"] | str]) -> None:
         """This is a special method that allow you to drop parts of the data model.
@@ -113,6 +216,26 @@ class DataModelPrepareAPI:
                 drops multiple views at once. You can also pass externalIds of views to drop individual views.
 
         """
-        if dms := self._state.last_verified_dms_rules:
-            output = ReduceCogniteModel(drop).transform(dms)
-            self._state.verified_rules.append(output.rules)
+        if input := self._state.data_model.last_verified_dms_rules:
+            source_id, rules = input
+            start = datetime.now(timezone.utc)
+
+            transformer = ReduceCogniteModel(drop)
+            output = transformer.transform(rules)
+            output.rules.metadata.version = f"{rules.metadata.version}.reduced"
+
+            end = datetime.now(timezone.utc)
+
+            change = Change.from_rules_activity(
+                output,
+                transformer.agent,
+                start,
+                end,
+                (
+                    f"Reduced data model {rules.metadata.as_data_model_id()}"
+                    f"on top of {rules.metadata.as_data_model_id()}"
+                ),
+                self._state.data_model.provenance.source_entity(source_id),
+            )
+
+            self._state.data_model.write(output.rules, change)

cognite/neat/_session/_read.py

@@ -1,8 +1,10 @@
+import tempfile
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
 
 from cognite.client import CogniteClient
-from cognite.client.data_classes.data_modeling import DataModelIdentifier
+from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
 
 from cognite.neat._graph import examples as instances_examples
 from cognite.neat._graph import extractors
@@ -10,10 +12,15 @@ from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._rules import importers
 from cognite.neat._rules._shared import ReadRules
+from cognite.neat._store._provenance import Activity as ProvenanceActivity
+from cognite.neat._store._provenance import Change
+from cognite.neat._store._provenance import Entity as ProvenanceEntity
+from cognite.neat._utils.reader import GitHubReader, NeatReader, PathReader
 
 from ._state import SessionState
 from ._wizard import NeatObjectType, RDFFileType, object_wizard, rdf_dm_wizard
-from .exceptions import intercept_session_exceptions
+from .engine import import_engine
+from .exceptions import NeatSessionError, intercept_session_exceptions
 
 
 @intercept_session_exceptions
@@ -24,6 +31,7 @@ class ReadAPI:
         self.cdf = CDFReadAPI(state, client, verbose)
         self.rdf = RDFReadAPI(state, client, verbose)
         self.excel = ExcelReadAPI(state, client, verbose)
+        self.csv = CSVReadAPI(state, client, verbose)
 
 
 @intercept_session_exceptions
@@ -33,14 +41,17 @@ class BaseReadAPI:
         self._verbose = verbose
         self._client = client
 
-    def _store_rules(self, io: Any, input_rules: ReadRules, source: str) -> None:
-        if input_rules.rules:
-            self._state.input_rules.append(input_rules)
-            if self._verbose:
-                if input_rules.issues.has_errors:
-                    print(f"{source} {type(io)} {io} read failed")
-                else:
-                    print(f"{source} {type(io)} {io} read successfully")
+    def _store_rules(self, rules: ReadRules, change: Change) -> IssueList:
+        if self._verbose:
+            if rules.issues.has_errors:
+                print("Data model read failed")
+            else:
+                print("Data model read passed")
+
+        if rules.rules:
+            self._state.data_model.write(rules, change)
+
+        return rules.issues
 
     def _return_filepath(self, io: Any) -> Path:
         if isinstance(io, str):
@@ -64,10 +75,36 @@ class CDFReadAPI(BaseReadAPI):
         return self._client
 
     def data_model(self, data_model_id: DataModelIdentifier) -> IssueList:
+        data_model_id = DataModelId.load(data_model_id)
+
+        if not data_model_id.version:
+            raise NeatSessionError("Data model version is required to read a data model.")
+
+        # actual reading of data model
+        start = datetime.now(timezone.utc)
         importer = importers.DMSImporter.from_data_model_id(self._get_client, data_model_id)
-        input_rules = importer.to_rules()
-        self._store_rules(data_model_id, input_rules, "CDF")
-        return input_rules.issues
+        rules = importer.to_rules()
+        end = datetime.now(timezone.utc)
+
+        # provenance information
+        source_entity = ProvenanceEntity.from_data_model_id(data_model_id)
+        agent = importer.agent
+        activity = ProvenanceActivity(
+            was_associated_with=agent,
+            ended_at_time=end,
+            used=source_entity,
+            started_at_time=start,
+        )
+        target_entity = ProvenanceEntity.from_rules(rules, agent, activity)
+        change = Change(
+            source_entity=source_entity,
+            agent=agent,
+            activity=activity,
+            target_entity=target_entity,
+            description=f"DMS Data model {data_model_id.as_tuple()} read as unverified data model",
+        )
+
+        return self._store_rules(rules, change)
 
 
 @intercept_session_exceptions
@@ -80,7 +117,7 @@ class CDFClassicAPI(BaseReadAPI):
 
     def assets(self, root_asset_external_id: str) -> None:
         extractor = extractors.AssetsExtractor.from_hierarchy(self._get_client, root_asset_external_id)
-        self._state.store.write(extractor)
+        self._state.instances.store.write(extractor)
         if self._verbose:
             print(f"Asset hierarchy {root_asset_external_id} read successfully")
 
@@ -88,28 +125,94 @@ class CDFClassicAPI(BaseReadAPI):
 @intercept_session_exceptions
 class ExcelReadAPI(BaseReadAPI):
     def __call__(self, io: Any) -> IssueList:
-        filepath = self._return_filepath(io)
-        input_rules: ReadRules = importers.ExcelImporter(filepath).to_rules()
-        self._store_rules(io, input_rules, "Excel")
+        reader = NeatReader.create(io)
+        start = datetime.now(timezone.utc)
+        if not isinstance(reader, PathReader):
+            raise NeatValueError("Only file paths are supported for Excel files")
+        importer: importers.ExcelImporter = importers.ExcelImporter(reader.path)
+        input_rules: ReadRules = importer.to_rules()
+        end = datetime.now(timezone.utc)
+
+        if input_rules.rules:
+            change = Change.from_rules_activity(
+                input_rules,
+                importer.agent,
+                start,
+                end,
+                description=f"Excel file {reader!s} read as unverified data model",
+            )
+            self._store_rules(input_rules, change)
+
         return input_rules.issues
 
 
+@intercept_session_exceptions
+class CSVReadAPI(BaseReadAPI):
+    def __call__(self, io: Any, type: str, primary_key: str) -> None:
+        reader = NeatReader.create(io)
+        if isinstance(reader, GitHubReader):
+            path = Path(tempfile.gettempdir()).resolve() / reader.name
+            path.write_text(reader.read_text())
+        elif isinstance(reader, PathReader):
+            path = reader.path
+        else:
+            raise NeatValueError("Only file paths are supported for CSV files")
+        engine = import_engine()
+        engine.set.source = ".csv"
+        engine.set.file = path
+        engine.set.type = type
+        engine.set.primary_key = primary_key
+        extractor = engine.create_extractor()
+
+        self._state.instances.store.write(extractor)
+
+
 @intercept_session_exceptions
 class RDFReadAPI(BaseReadAPI):
     def __init__(self, state: SessionState, client: CogniteClient | None, verbose: bool) -> None:
         super().__init__(state, client, verbose)
         self.examples = RDFExamples(state)
 
-    def _ontology(self, io: Any) -> IssueList:
-        filepath = self._return_filepath(io)
-        input_rules: ReadRules = importers.OWLImporter.from_file(filepath).to_rules()
-        self._store_rules(io, input_rules, "Ontology")
+    def ontology(self, io: Any) -> IssueList:
+        start = datetime.now(timezone.utc)
+        reader = NeatReader.create(io)
+        if not isinstance(reader, PathReader):
+            raise NeatValueError("Only file paths are supported for RDF files")
+        importer = importers.OWLImporter.from_file(reader.path)
+        input_rules: ReadRules = importer.to_rules()
+        end = datetime.now(timezone.utc)
+
+        if input_rules.rules:
+            change = Change.from_rules_activity(
+                input_rules,
+                importer.agent,
+                start,
+                end,
+                description=f"Ontology file {reader!s} read as unverified data model",
+            )
+            self._store_rules(input_rules, change)
+
         return input_rules.issues
 
-    def _imf(self, io: Any) -> IssueList:
-        filepath = self._return_filepath(io)
-        input_rules: ReadRules = importers.IMFImporter.from_file(filepath).to_rules()
-        self._store_rules(io, input_rules, "IMF Types")
+    def imf(self, io: Any) -> IssueList:
+        start = datetime.now(timezone.utc)
+        reader = NeatReader.create(io)
+        if not isinstance(reader, PathReader):
+            raise NeatValueError("Only file paths are supported for RDF files")
+        importer = importers.IMFImporter.from_file(reader.path)
+        input_rules: ReadRules = importer.to_rules()
+        end = datetime.now(timezone.utc)
+
+        if input_rules.rules:
+            change = Change.from_rules_activity(
+                input_rules,
+                importer.agent,
+                start,
+                end,
+                description=f"IMF Types file {reader!s} read as unverified data model",
+            )
+            self._store_rules(input_rules, change)
+
         return input_rules.issues
 
     def __call__(
@@ -124,17 +227,21 @@ class RDFReadAPI(BaseReadAPI):
         if type.lower() == "Data Model".lower():
             source = source or rdf_dm_wizard("What type of data model is the RDF?")
             if source == "Ontology":
-                return self._ontology(io)
+                return self.ontology(io)
             elif source == "IMF":
-                return self._imf(io)
+                return self.imf(io)
             else:
                 raise ValueError(f"Expected ontology, imf or instances, got {source}")
 
         elif type.lower() == "Instances".lower():
-            self._state.store.write(extractors.RdfFileExtractor(self._return_filepath(io)))
+            reader = NeatReader.create(io)
+            if not isinstance(reader, PathReader):
+                raise NeatValueError("Only file paths are supported for RDF files")
+
+            self._state.instances.store.write(extractors.RdfFileExtractor(reader.path))
             return IssueList()
         else:
-            raise ValueError(f"Expected data model or instances, got {type}")
+            raise NeatSessionError(f"Expected data model or instances, got {type}")
 
 
 class RDFExamples:
@@ -143,5 +250,5 @@ class RDFExamples:
 
     @property
     def nordic44(self) -> IssueList:
-        self._state.store.write(extractors.RdfFileExtractor(instances_examples.nordic44_knowledge_graph))
+        self._state.instances.store.write(extractors.RdfFileExtractor(instances_examples.nordic44_knowledge_graph))
         return IssueList()

cognite/neat/_session/_set.py

@@ -1,6 +1,9 @@
+from datetime import datetime, timezone
+
 from cognite.client import data_modeling as dm
 
 from cognite.neat._rules.transformers import SetIDDMSModel
+from cognite.neat._store._provenance import Change
 
 from ._state import SessionState
 from .exceptions import intercept_session_exceptions
@@ -14,9 +17,25 @@ class SetAPI:
 
     def data_model_id(self, new_model_id: dm.DataModelId | tuple[str, str, str]) -> None:
         """Sets the data model ID of the latest verified data model."""
-        if dms := self._state.last_verified_dms_rules:
-            output = SetIDDMSModel(new_model_id).transform(dms)
-            self._state.verified_rules.append(output.rules)
+        if res := self._state.data_model.last_verified_dms_rules:
+            source_id, rules = res
+
+            start = datetime.now(timezone.utc)
+            transformer = SetIDDMSModel(new_model_id)
+            output = transformer.transform(rules)
+            end = datetime.now(timezone.utc)
+
+            # Provenance
+            change = Change.from_rules_activity(
+                output,
+                transformer.agent,
+                start,
+                end,
+                "Changed data model id",
+                self._state.data_model.provenance.source_entity(source_id),
+            )
+
+            self._state.data_model.write(output, change)
             if self._verbose:
                 print(f"Data model ID set to {new_model_id}")
         else: