cognite-neat 0.85.6__py3-none-any.whl → 0.85.8__py3-none-any.whl

cognite/neat/graph/extractors/_classic_cdf/_labels.py

@@ -37,7 +37,11 @@ class LabelsExtractor(BaseExtractor):
         namespace: Namespace | None = None,
     ):
         return cls(
-            cast(Iterable[LabelDefinition], client.labels(data_set_external_ids=data_set_external_id)), namespace
+            cast(
+                Iterable[LabelDefinition],
+                client.labels(data_set_external_ids=data_set_external_id),
+            ),
+            namespace,
         )
 
     @classmethod
@@ -47,36 +51,41 @@ class LabelsExtractor(BaseExtractor):
     def extract(self) -> Iterable[Triple]:
         """Extract labels as triples."""
         for label in self.labels:
-            yield from self._labels2triples(label, self.namespace)
+            yield from self._labels2triples(label)
 
-    @classmethod
-    def _labels2triples(cls, label: LabelDefinition, namespace: Namespace) -> list[Triple]:
+    def _labels2triples(self, label: LabelDefinition) -> list[Triple]:
         if label.external_id:
-            id_ = namespace[f"Label_{create_sha256_hash(label.external_id)}"]
+            id_ = self.namespace[f"Label_{create_sha256_hash(label.external_id)}"]
 
             # Set rdf type
-            triples: list[Triple] = [(id_, RDF.type, namespace.Label)]
+            triples: list[Triple] = [(id_, RDF.type, self.namespace.Label)]
 
             # Create attributes
-            triples.append((id_, namespace.external_id, Literal(label.external_id)))
+            triples.append((id_, self.namespace.external_id, Literal(label.external_id)))
 
             if label.name:
-                triples.append((id_, namespace.name, Literal(label.name)))
+                triples.append((id_, self.namespace.name, Literal(label.name)))
 
             if label.description:
-                triples.append((id_, namespace.description, Literal(label.description)))
+                triples.append((id_, self.namespace.description, Literal(label.description)))
 
             if label.created_time:
                 triples.append(
                     (
                         id_,
-                        namespace.created_time,
+                        self.namespace.created_time,
                         Literal(datetime.fromtimestamp(label.created_time / 1000, timezone.utc)),
                     )
                 )
 
             if label.data_set_id:
-                triples.append((id_, namespace.data_set_id, namespace[f"Dataset_{label.data_set_id}"]))
+                triples.append(
+                    (
+                        id_,
+                        self.namespace.data_set_id,
+                        self.namespace[f"Dataset_{label.data_set_id}"],
+                    )
+                )
 
             return triples
         return []

cognite/neat/graph/extractors/_classic_cdf/_relationships.py

@@ -38,7 +38,11 @@ class RelationshipsExtractor(BaseExtractor):
         namespace: Namespace | None = None,
     ):
         return cls(
-            cast(Iterable[Relationship], client.relationships(data_set_external_ids=data_set_external_id)), namespace
+            cast(
+                Iterable[Relationship],
+                client.relationships(data_set_external_ids=data_set_external_id),
+            ),
+            namespace,
         )
 
     @classmethod
@@ -48,26 +52,25 @@ class RelationshipsExtractor(BaseExtractor):
     def extract(self) -> Iterable[Triple]:
         """Extracts an asset with the given asset_id."""
         for relationship in self.relationships:
-            yield from self._relationship2triples(relationship, self.namespace)
+            yield from self._relationship2triples(relationship)
 
-    @classmethod
-    def _relationship2triples(cls, relationship: Relationship, namespace: Namespace) -> list[Triple]:
+    def _relationship2triples(self, relationship: Relationship) -> list[Triple]:
         """Converts an asset to triples."""
 
         if relationship.external_id and relationship.source_external_id and relationship.target_external_id:
             # relationships do not have an internal id, so we generate one
-            id_ = namespace[f"Relationship_{create_sha256_hash(relationship.external_id)}"]
+            id_ = self.namespace[f"Relationship_{create_sha256_hash(relationship.external_id)}"]
 
             # Set rdf type
-            triples: list[Triple] = [(id_, RDF.type, namespace["Relationship"])]
+            triples: list[Triple] = [(id_, RDF.type, self.namespace["Relationship"])]
 
             # Set source and target types
             if source_type := relationship.source_type:
                 triples.append(
                     (
                         id_,
-                        namespace.source_type,
-                        namespace[source_type.title()],
+                        self.namespace.source_type,
+                        self.namespace[source_type.title()],
                     )
                 )
 
@@ -75,19 +78,19 @@ class RelationshipsExtractor(BaseExtractor):
                 triples.append(
                     (
                         id_,
-                        namespace.target_type,
-                        namespace[target_type.title()],
+                        self.namespace.target_type,
+                        self.namespace[target_type.title()],
                     )
                 )
 
             # Create attributes
 
-            triples.append((id_, namespace.external_id, Literal(relationship.external_id)))
+            triples.append((id_, self.namespace.external_id, Literal(relationship.external_id)))
 
             triples.append(
                 (
                     id_,
-                    namespace.source_external_id,
+                    self.namespace.source_external_id,
                     Literal(relationship.source_external_id),
                 )
             )
@@ -95,7 +98,7 @@ class RelationshipsExtractor(BaseExtractor):
             triples.append(
                 (
                     id_,
-                    namespace.target_external_id,
+                    self.namespace.target_external_id,
                     Literal(relationship.target_external_id),
                 )
             )
@@ -104,7 +107,7 @@ class RelationshipsExtractor(BaseExtractor):
                 triples.append(
                     (
                         id_,
-                        namespace.start_time,
+                        self.namespace.start_time,
                         Literal(datetime.fromtimestamp(relationship.start_time / 1000, timezone.utc)),
                     )
                 )
@@ -113,7 +116,7 @@ class RelationshipsExtractor(BaseExtractor):
                 triples.append(
                     (
                         id_,
-                        namespace.end_time,
+                        self.namespace.end_time,
                         Literal(datetime.fromtimestamp(relationship.end_time / 1000, timezone.utc)),
                     )
                 )
@@ -122,7 +125,7 @@ class RelationshipsExtractor(BaseExtractor):
                 triples.append(
                     (
                         id_,
-                        namespace.created_time,
+                        self.namespace.created_time,
                         Literal(datetime.fromtimestamp(relationship.created_time / 1000, timezone.utc)),
                     )
                 )
@@ -131,7 +134,7 @@ class RelationshipsExtractor(BaseExtractor):
                 triples.append(
                     (
                         id_,
-                        namespace.last_updated_time,
+                        self.namespace.last_updated_time,
                         Literal(datetime.fromtimestamp(relationship.last_updated_time / 1000, timezone.utc)),
                     )
                 )
@@ -140,7 +143,7 @@ class RelationshipsExtractor(BaseExtractor):
                 triples.append(
                     (
                         id_,
-                        namespace.confidence,
+                        self.namespace.confidence,
                         Literal(relationship.confidence),
                     )
                 )
@@ -149,11 +152,23 @@ class RelationshipsExtractor(BaseExtractor):
                 for label in relationship.labels:
                     # external_id can create ill-formed URIs, so we create websafe URIs
                     # since labels do not have internal ids, we use the external_id as the id
-                    triples.append((id_, namespace.label, namespace[f"Label_{quote(label.dump()['externalId'])}"]))
+                    triples.append(
+                        (
+                            id_,
+                            self.namespace.label,
+                            self.namespace[f"Label_{quote(label.dump()['externalId'])}"],
+                        )
+                    )
 
             # Create connection
             if relationship.data_set_id:
-                triples.append((id_, namespace.dataset, namespace[f"Dataset_{relationship.data_set_id}"]))
+                triples.append(
+                    (
+                        id_,
+                        self.namespace.dataset,
+                        self.namespace[f"Dataset_{relationship.data_set_id}"],
+                    )
+                )
 
             return triples
         return []

cognite/neat/graph/extractors/_classic_cdf/_sequences.py

@@ -1,3 +1,4 @@
+import json
 from collections.abc import Iterable
 from datetime import datetime, timezone
 from pathlib import Path
@@ -20,15 +21,19 @@ class SequencesExtractor(BaseExtractor):
     Args:
         sequence (Iterable[Sequence]): An iterable of sequences.
         namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
+        unpack_metadata (bool, optional): Whether to unpack metadata. Defaults to False, which yields the metadata as
+            a JSON string.
     """
 
     def __init__(
         self,
         sequence: Iterable[Sequence],
         namespace: Namespace | None = None,
+        unpack_metadata: bool = False,
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
         self.sequence = sequence
+        self.unpack_metadata = unpack_metadata
 
     @classmethod
     def from_dataset(
@@ -36,50 +41,71 @@ class SequencesExtractor(BaseExtractor):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
+        unpack_metadata: bool = False,
     ):
-        return cls(cast(Iterable[Sequence], client.sequences(data_set_external_ids=data_set_external_id)), namespace)
+        return cls(
+            cast(
+                Iterable[Sequence],
+                client.sequences(data_set_external_ids=data_set_external_id),
+            ),
+            namespace,
+            unpack_metadata,
+        )
 
     @classmethod
-    def from_file(cls, file_path: str, namespace: Namespace | None = None):
-        return cls(SequenceList.load(Path(file_path).read_text()), namespace)
+    def from_file(
+        cls,
+        file_path: str,
+        namespace: Namespace | None = None,
+        unpack_metadata: bool = False,
+    ):
+        return cls(SequenceList.load(Path(file_path).read_text()), namespace, unpack_metadata)
 
     def extract(self) -> Iterable[Triple]:
         """Extract sequences as triples."""
         for sequence in self.sequence:
-            yield from self._sequence2triples(sequence, self.namespace)
+            yield from self._sequence2triples(sequence)
 
-    @classmethod
-    def _sequence2triples(cls, sequence: Sequence, namespace: Namespace) -> list[Triple]:
-        id_ = namespace[f"Sequence_{sequence.id}"]
+    def _sequence2triples(self, sequence: Sequence) -> list[Triple]:
+        id_ = self.namespace[f"Sequence_{sequence.id}"]
 
         # Set rdf type
-        triples: list[Triple] = [(id_, RDF.type, namespace.Sequence)]
+        triples: list[Triple] = [(id_, RDF.type, self.namespace.Sequence)]
 
         # Create attributes
 
         if sequence.external_id:
-            triples.append((id_, namespace.external_id, Literal(sequence.external_id)))
+            triples.append((id_, self.namespace.external_id, Literal(sequence.external_id)))
 
         if sequence.name:
-            triples.append((id_, namespace.name, Literal(sequence.name)))
+            triples.append((id_, self.namespace.name, Literal(sequence.name)))
 
         if sequence.metadata:
-            for key, value in sequence.metadata.items():
-                if value:
-                    type_aware_value = string_to_ideal_type(value)
-                    try:
-                        triples.append((id_, namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
-                    except ValidationError:
-                        triples.append((id_, namespace[key], Literal(type_aware_value)))
+            if self.unpack_metadata:
+                for key, value in sequence.metadata.items():
+                    if value:
+                        type_aware_value = string_to_ideal_type(value)
+                        try:
+                            triples.append((id_, self.namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
+                        except ValidationError:
+                            triples.append((id_, self.namespace[key], Literal(type_aware_value)))
+            else:
+                triples.append(
+                    (
+                        id_,
+                        self.namespace.metadata,
+                        Literal(json.dumps(sequence.metadata)),
+                    )
+                )
 
         if sequence.description:
-            triples.append((id_, namespace.description, Literal(sequence.description)))
+            triples.append((id_, self.namespace.description, Literal(sequence.description)))
 
         if sequence.created_time:
             triples.append(
                 (
                     id_,
-                    namespace.created_time,
+                    self.namespace.created_time,
                     Literal(datetime.fromtimestamp(sequence.created_time / 1000, timezone.utc)),
                 )
             )
@@ -88,15 +114,27 @@ class SequencesExtractor(BaseExtractor):
             triples.append(
                 (
                     id_,
-                    namespace.last_updated_time,
+                    self.namespace.last_updated_time,
                     Literal(datetime.fromtimestamp(sequence.last_updated_time / 1000, timezone.utc)),
                 )
             )
 
         if sequence.data_set_id:
-            triples.append((id_, namespace.data_set_id, namespace[f"Dataset_{sequence.data_set_id}"]))
+            triples.append(
+                (
+                    id_,
+                    self.namespace.data_set_id,
+                    self.namespace[f"Dataset_{sequence.data_set_id}"],
+                )
+            )
 
         if sequence.asset_id:
-            triples.append((id_, namespace.asset, namespace[f"Asset_{sequence.asset_id}"]))
+            triples.append(
+                (
+                    id_,
+                    self.namespace.asset,
+                    self.namespace[f"Asset_{sequence.asset_id}"],
+                )
+            )
 
         return triples

cognite/neat/graph/extractors/_classic_cdf/_timeseries.py

@@ -1,3 +1,4 @@
+import json
 from collections.abc import Iterable
 from datetime import datetime, timezone
 from pathlib import Path
@@ -20,15 +21,19 @@ class TimeSeriesExtractor(BaseExtractor):
     Args:
         timeseries (Iterable[TimeSeries]): An iterable of timeseries.
         namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
+        unpack_metadata (bool, optional): Whether to unpack metadata. Defaults to False, which yields the metadata as
+            a JSON string.
     """
 
     def __init__(
         self,
         timeseries: Iterable[TimeSeries],
         namespace: Namespace | None = None,
+        unpack_metadata: bool = False,
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
         self.timeseries = timeseries
+        self.unpack_metadata = unpack_metadata
 
     @classmethod
     def from_dataset(
@@ -36,65 +41,84 @@ class TimeSeriesExtractor(BaseExtractor):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
+        unpack_metadata: bool = False,
     ):
         return cls(
-            cast(Iterable[TimeSeries], client.time_series(data_set_external_ids=data_set_external_id)), namespace
+            cast(
+                Iterable[TimeSeries],
+                client.time_series(data_set_external_ids=data_set_external_id),
+            ),
+            namespace,
+            unpack_metadata,
         )
 
     @classmethod
-    def from_file(cls, file_path: str, namespace: Namespace | None = None):
-        return cls(TimeSeriesList.load(Path(file_path).read_text()), namespace)
+    def from_file(
+        cls,
+        file_path: str,
+        namespace: Namespace | None = None,
+        unpack_metadata: bool = False,
+    ):
+        return cls(TimeSeriesList.load(Path(file_path).read_text()), namespace, unpack_metadata)
 
     def extract(self) -> Iterable[Triple]:
         """Extract timeseries as triples."""
         for timeseries in self.timeseries:
-            yield from self._timeseries2triples(timeseries, self.namespace)
+            yield from self._timeseries2triples(timeseries)
 
-    @classmethod
-    def _timeseries2triples(cls, timeseries: TimeSeries, namespace: Namespace) -> list[Triple]:
-        id_ = namespace[f"TimeSeries_{timeseries.id}"]
+    def _timeseries2triples(self, timeseries: TimeSeries) -> list[Triple]:
+        id_ = self.namespace[f"TimeSeries_{timeseries.id}"]
 
         # Set rdf type
-        triples: list[Triple] = [(id_, RDF.type, namespace.TimeSeries)]
+        triples: list[Triple] = [(id_, RDF.type, self.namespace.TimeSeries)]
 
         # Create attributes
 
         if timeseries.external_id:
-            triples.append((id_, namespace.external_id, Literal(timeseries.external_id)))
+            triples.append((id_, self.namespace.external_id, Literal(timeseries.external_id)))
 
         if timeseries.name:
-            triples.append((id_, namespace.name, Literal(timeseries.name)))
+            triples.append((id_, self.namespace.name, Literal(timeseries.name)))
 
         if timeseries.is_string:
-            triples.append((id_, namespace.is_string, Literal(timeseries.is_string)))
+            triples.append((id_, self.namespace.is_string, Literal(timeseries.is_string)))
 
         if timeseries.metadata:
-            for key, value in timeseries.metadata.items():
-                if value:
-                    type_aware_value = string_to_ideal_type(value)
-                    try:
-                        triples.append((id_, namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
-                    except ValidationError:
-                        triples.append((id_, namespace[key], Literal(type_aware_value)))
+            if self.unpack_metadata:
+                for key, value in timeseries.metadata.items():
+                    if value:
+                        type_aware_value = string_to_ideal_type(value)
+                        try:
+                            triples.append((id_, self.namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
+                        except ValidationError:
+                            triples.append((id_, self.namespace[key], Literal(type_aware_value)))
+            else:
+                triples.append(
+                    (
+                        id_,
+                        self.namespace.metadata,
+                        Literal(json.dumps(timeseries.metadata)),
+                    )
+                )
 
         if timeseries.unit:
-            triples.append((id_, namespace.unit, Literal(timeseries.unit)))
+            triples.append((id_, self.namespace.unit, Literal(timeseries.unit)))
 
-        if namespace.is_step:
-            triples.append((id_, namespace.is_step, Literal(timeseries.is_step)))
+        if self.namespace.is_step:
+            triples.append((id_, self.namespace.is_step, Literal(timeseries.is_step)))
 
         if timeseries.description:
-            triples.append((id_, namespace.description, Literal(timeseries.description)))
+            triples.append((id_, self.namespace.description, Literal(timeseries.description)))
 
         if timeseries.security_categories:
             for category in timeseries.security_categories:
-                triples.append((id_, namespace.security_categories, Literal(category)))
+                triples.append((id_, self.namespace.security_categories, Literal(category)))
 
         if timeseries.created_time:
             triples.append(
                 (
                     id_,
-                    namespace.created_time,
+                    self.namespace.created_time,
                     Literal(datetime.fromtimestamp(timeseries.created_time / 1000, timezone.utc)),
                 )
             )
@@ -103,26 +127,50 @@ class TimeSeriesExtractor(BaseExtractor):
             triples.append(
                 (
                     id_,
-                    namespace.last_updated_time,
+                    self.namespace.last_updated_time,
                     Literal(datetime.fromtimestamp(timeseries.last_updated_time / 1000, timezone.utc)),
                 )
             )
 
         if timeseries.legacy_name:
-            triples.append((id_, namespace.legacy_name, Literal(timeseries.legacy_name)))
+            triples.append((id_, self.namespace.legacy_name, Literal(timeseries.legacy_name)))
 
         # Create connections
         if timeseries.unit_external_id:
             # try to create connection to QUDT unit catalog
             try:
-                triples.append((id_, namespace.unit_external_id, URIRef(str(AnyHttpUrl(timeseries.unit_external_id)))))
+                triples.append(
+                    (
+                        id_,
+                        self.namespace.unit_external_id,
+                        URIRef(str(AnyHttpUrl(timeseries.unit_external_id))),
+                    )
+                )
             except ValidationError:
-                triples.append((id_, namespace.unit_external_id, Literal(timeseries.unit_external_id)))
+                triples.append(
+                    (
+                        id_,
+                        self.namespace.unit_external_id,
+                        Literal(timeseries.unit_external_id),
+                    )
+                )
 
         if timeseries.data_set_id:
-            triples.append((id_, namespace.dataset, namespace[f"Dataset_{timeseries.data_set_id}"]))
+            triples.append(
+                (
+                    id_,
+                    self.namespace.dataset,
+                    self.namespace[f"Dataset_{timeseries.data_set_id}"],
+                )
+            )
 
         if timeseries.asset_id:
-            triples.append((id_, namespace.asset, namespace[f"Asset_{timeseries.asset_id}"]))
+            triples.append(
+                (
+                    id_,
+                    self.namespace.asset,
+                    self.namespace[f"Asset_{timeseries.asset_id}"],
+                )
+            )
 
         return triples

cognite/neat/graph/extractors/_mock_graph_generator.py

@@ -17,7 +17,7 @@ from cognite.neat.rules.models import DMSRules, InformationRules
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
 from cognite.neat.rules.models.information import InformationProperty
-from cognite.neat.utils.utils import remove_namespace
+from cognite.neat.utils.utils import remove_namespace_from_uri
 
 from ._base import BaseExtractor
 
@@ -258,7 +258,11 @@ def _generate_mock_data_property_triples(
         # generate string
         else:
             triples.append(
-                (id_, URIRef(namespace[property_]), Literal(f"{property_}-{remove_namespace(id_).split('-')[-1]}"))
+                (
+                    id_,
+                    URIRef(namespace[property_]),
+                    Literal(f"{property_}-{remove_namespace_from_uri(id_).split('-')[-1]}"),
+                )
             )
     return triples
 

cognite/neat/graph/loaders/_base.py

@@ -65,11 +65,11 @@ class CDFLoader(BaseLoader[T_Output]):
                 items.append(result)
 
             if len(items) >= self._UPLOAD_BATCH_SIZE:
-                yield self._upload_to_cdf(client, items, dry_run, issues)
+                yield from self._upload_to_cdf(client, items, dry_run, issues)
                 issues = NeatIssueList[NeatIssue]()
                 items = []
         if items:
-            yield self._upload_to_cdf(client, items, dry_run, issues)
+            yield from self._upload_to_cdf(client, items, dry_run, issues)
 
     @abstractmethod
     def _get_required_capabilities(self) -> list[Capability]:
@@ -82,5 +82,5 @@ class CDFLoader(BaseLoader[T_Output]):
         items: list[T_Output],
         dry_run: bool,
         read_issues: NeatIssueList,
-    ) -> UploadResult:
+    ) -> Iterable[UploadResult]:
         raise NotImplementedError

cognite/neat/graph/loaders/_rdf2dms.py

@@ -1,4 +1,3 @@
-import itertools
 import json
 from collections import defaultdict
 from collections.abc import Iterable, Sequence
@@ -263,8 +262,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         items: list[dm.InstanceApply],
         dry_run: bool,
         read_issues: NeatIssueList,
-    ) -> UploadResult:
-        result = UploadResult[InstanceId](name=type(self).__name__, issues=read_issues)
+    ) -> Iterable[UploadResult]:
         try:
             nodes = [item for item in items if isinstance(item, dm.NodeApply)]
             edges = [item for item in items if isinstance(item, dm.EdgeApply)]
@@ -276,18 +274,22 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                 skip_on_version_conflict=True,
             )
         except CogniteAPIError as e:
+            result = UploadResult[InstanceId](name="Instances", issues=read_issues)
             result.error_messages.append(str(e))
             result.failed_upserted.update(item.as_id() for item in e.failed + e.unknown)
             result.created.update(item.as_id() for item in e.successful)
+            yield result
         else:
-            for instance in itertools.chain(upserted.nodes, upserted.edges):
-                if instance.was_modified and instance.created_time == instance.last_updated_time:
-                    result.created.add(instance.as_id())
-                elif instance.was_modified:
-                    result.changed.add(instance.as_id())
-                else:
-                    result.unchanged.add(instance.as_id())
-        return result
+            for instance_type, instances in {"Nodes": upserted.nodes, "Edges": upserted.edges}.items():
+                result = UploadResult[InstanceId](name=instance_type, issues=read_issues)
+                for instance in instances:  # type: ignore[attr-defined]
+                    if instance.was_modified and instance.created_time == instance.last_updated_time:
+                        result.created.add(instance.as_id())
+                    elif instance.was_modified:
+                        result.changed.add(instance.as_id())
+                    else:
+                        result.unchanged.add(instance.as_id())
+                yield result
 
 
 def _triples2dictionary(