cognite-neat 0.85.6__py3-none-any.whl → 0.85.8__py3-none-any.whl

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.85.6"
+__version__ = "0.85.8"

cognite/neat/app/api/routers/data_exploration.py

@@ -17,7 +17,7 @@ from cognite.neat.app.api.data_classes.rest import (
 from cognite.neat.app.api.utils.data_mapping import rdf_result_to_api_response
 from cognite.neat.app.api.utils.query_templates import query_templates
 from cognite.neat.legacy.graph.transformations import query_generator
-from cognite.neat.utils.utils import remove_namespace
+from cognite.neat.utils.utils import remove_namespace_from_uri
 from cognite.neat.workflows.steps.data_contracts import RulesData, SolutionGraph, SourceGraph
 
 router = APIRouter()
@@ -51,7 +51,7 @@ def get_datatype_properties(request: DatatypePropertyRequest):
             {
                 "id": row[rdflib.Variable("property")],
                 "count": int(row[rdflib.Variable("occurrence")]),
-                "name": remove_namespace(row[rdflib.Variable("property")]),
+                "name": remove_namespace_from_uri(row[rdflib.Variable("property")]),
             }
             for row in results["rows"]
         ]

cognite/neat/graph/extractors/_classic_cdf/_assets.py

@@ -1,10 +1,12 @@
-from collections.abc import Iterable
+import json
+import re
+from collections.abc import Callable, Iterable
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import cast
 
 from cognite.client import CogniteClient
-from cognite.client.data_classes import Asset, AssetList
+from cognite.client.data_classes import Asset, AssetFilter, AssetList
 from rdflib import RDF, Literal, Namespace
 
 from cognite.neat.constants import DEFAULT_NAMESPACE
@@ -19,15 +21,34 @@ class AssetsExtractor(BaseExtractor):
     Args:
         assets (Iterable[Asset]): An iterable of assets.
         namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
+        to_type (Callable[[Asset], str | None], optional): A function to convert an asset to a type. Defaults to None.
+            If None or if the function returns None, the asset will be set to the default type "Asset".
+        total (int, optional): The total number of assets to load. If passed, you will get a progress bar if rich
+            is installed. Defaults to None.
+        limit (int, optional): The maximal number of assets to load. Defaults to None. This is typically used for
+            testing setup of the extractor. For example, if you are extracting 100 000 assets, you might want to
+            limit the extraction to 1000 assets to test the setup.
+        unpack_metadata (bool, optional): Whether to unpack metadata. Defaults to False, which yields the metadata as
+            a JSON string.
     """
 
+    _SPACE_PATTERN = re.compile(r"\s+")
+
     def __init__(
         self,
         assets: Iterable[Asset],
         namespace: Namespace | None = None,
+        to_type: Callable[[Asset], str | None] | None = None,
+        total: int | None = None,
+        limit: int | None = None,
+        unpack_metadata: bool = False,
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
         self.assets = assets
+        self.to_type = to_type
+        self.total = total
+        self.limit = min(limit, total) if limit and total else limit
+        self.unpack_metadata = unpack_metadata
 
     @classmethod
     def from_dataset(
@@ -35,55 +56,124 @@ class AssetsExtractor(BaseExtractor):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
+        to_type: Callable[[Asset], str | None] | None = None,
+        limit: int | None = None,
+        unpack_metadata: bool = False,
     ):
-        return cls(cast(Iterable[Asset], client.assets(data_set_external_ids=data_set_external_id)), namespace)
+        total = client.assets.aggregate_count(filter=AssetFilter(data_set_ids=[{"externalId": data_set_external_id}]))
+
+        return cls(
+            cast(
+                Iterable[Asset],
+                client.assets(data_set_external_ids=data_set_external_id),
+            ),
+            namespace,
+            to_type,
+            total,
+            limit,
+            unpack_metadata=unpack_metadata,
+        )
 
     @classmethod
-    def from_hierarchy(cls, client: CogniteClient, root_asset_external_id: str, namespace: Namespace | None = None):
-        return cls(cast(Iterable[Asset], client.assets(asset_subtree_external_ids=root_asset_external_id)), namespace)
+    def from_hierarchy(
+        cls,
+        client: CogniteClient,
+        root_asset_external_id: str,
+        namespace: Namespace | None = None,
+        to_type: Callable[[Asset], str | None] | None = None,
+        limit: int | None = None,
+        unpack_metadata: bool = False,
+    ):
+        total = client.assets.aggregate_count(
+            filter=AssetFilter(asset_subtree_ids=[{"externalId": root_asset_external_id}])
+        )
+
+        return cls(
+            cast(
+                Iterable[Asset],
+                client.assets(asset_subtree_external_ids=root_asset_external_id),
+            ),
+            namespace,
+            to_type,
+            total,
+            limit,
+            unpack_metadata=unpack_metadata,
+        )
 
     @classmethod
-    def from_file(cls, file_path: str, namespace: Namespace | None = None):
-        return cls(AssetList.load(Path(file_path).read_text()), namespace)
+    def from_file(
+        cls,
+        file_path: str,
+        namespace: Namespace | None = None,
+        to_type: Callable[[Asset], str] | None = None,
+        limit: int | None = None,
+        unpack_metadata: bool = False,
+    ):
+        return cls(
+            AssetList.load(Path(file_path).read_text()),
+            namespace,
+            to_type,
+            limit,
+            unpack_metadata=unpack_metadata,
+        )
 
     def extract(self) -> Iterable[Triple]:
         """Extracts an asset with the given asset_id."""
-        for asset in self.assets:
-            yield from self._asset2triples(asset, self.namespace)
-
-    @classmethod
-    def _asset2triples(cls, asset: Asset, namespace: Namespace) -> list[Triple]:
+        if self.total:
+            try:
+                from rich.progress import track
+            except ModuleNotFoundError:
+                to_iterate = self.assets
+            else:
+                to_iterate = track(
+                    self.assets,
+                    total=self.limit or self.total,
+                    description="Extracting Assets",
+                )
+        else:
+            to_iterate = self.assets
+        for no, asset in enumerate(to_iterate):
+            yield from self._asset2triples(asset)
+            if self.limit and no >= self.limit:
+                break
+
+    def _asset2triples(self, asset: Asset) -> list[Triple]:
         """Converts an asset to triples."""
-        id_ = namespace[f"Asset_{asset.id}"]
+        id_ = self.namespace[f"Asset_{asset.id}"]
 
         # Set rdf type
-        triples: list[Triple] = [(id_, RDF.type, namespace["Asset"])]
+        type_ = "Asset"
+        if self.to_type:
+            type_ = self.to_type(asset) or type_
+        type_ = self._SPACE_PATTERN.sub("_", type_)
+
+        triples: list[Triple] = [(id_, RDF.type, self.namespace[type_])]
 
         # Create attributes
         if asset.name:
-            triples.append((id_, namespace.name, Literal(asset.name)))
+            triples.append((id_, self.namespace.name, Literal(asset.name)))
 
         if asset.description:
-            triples.append((id_, namespace.description, Literal(asset.description)))
+            triples.append((id_, self.namespace.description, Literal(asset.description)))
 
         if asset.external_id:
-            triples.append((id_, namespace.external_id, Literal(asset.external_id)))
+            triples.append((id_, self.namespace.external_id, Literal(asset.external_id)))
 
         if asset.source:
-            triples.append((id_, namespace.source, Literal(asset.source)))
+            triples.append((id_, self.namespace.source, Literal(asset.source)))
 
         # properties ref creation and update
         triples.append(
             (
                 id_,
-                namespace.created_time,
+                self.namespace.created_time,
                 Literal(datetime.fromtimestamp(asset.created_time / 1000, timezone.utc)),
             )
         )
         triples.append(
             (
                 id_,
-                namespace.last_updated_time,
+                self.namespace.last_updated_time,
                 Literal(datetime.fromtimestamp(asset.last_updated_time / 1000, timezone.utc)),
             )
         )
@@ -93,22 +183,41 @@ class AssetsExtractor(BaseExtractor):
                 # external_id can create ill-formed URIs, so we create websafe URIs
                 # since labels do not have internal ids, we use the external_id as the id
                 triples.append(
-                    (id_, namespace.label, namespace[f"Label_{create_sha256_hash(label.dump()['externalId'])}"])
+                    (
+                        id_,
+                        self.namespace.label,
+                        self.namespace[f"Label_{create_sha256_hash(label.dump()['externalId'])}"],
+                    )
                 )
 
         if asset.metadata:
-            for key, value in asset.metadata.items():
-                if value:
-                    triples.append((id_, namespace[key], Literal(string_to_ideal_type(value))))
+            if self.unpack_metadata:
+                for key, value in asset.metadata.items():
+                    if value:
+                        triples.append(
+                            (
+                                id_,
+                                self.namespace[key],
+                                Literal(string_to_ideal_type(value)),
+                            )
+                        )
+            else:
+                triples.append((id_, self.namespace.metadata, Literal(json.dumps(asset.metadata))))
 
         # Create connections:
         if asset.parent_id:
-            triples.append((id_, namespace.parent, namespace[f"Asset_{asset.parent_id}"]))
+            triples.append((id_, self.namespace.parent, self.namespace[f"Asset_{asset.parent_id}"]))
 
         if asset.root_id:
-            triples.append((id_, namespace.root, namespace[f"Asset_{asset.root_id}"]))
+            triples.append((id_, self.namespace.root, self.namespace[f"Asset_{asset.root_id}"]))
 
         if asset.data_set_id:
-            triples.append((id_, namespace.dataset, namespace[f"Dataset_{asset.data_set_id}"]))
+            triples.append(
+                (
+                    id_,
+                    self.namespace.dataset,
+                    self.namespace[f"Dataset_{asset.data_set_id}"],
+                )
+            )
 
         return triples

cognite/neat/graph/extractors/_classic_cdf/_events.py

@@ -1,3 +1,4 @@
+import json
 from collections.abc import Iterable
 from datetime import datetime, timezone
 from pathlib import Path
@@ -20,15 +21,19 @@ class EventsExtractor(BaseExtractor):
     Args:
         events (Iterable[Event]): An iterable of events.
         namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
+        unpack_metadata (bool, optional): Whether to unpack metadata. Defaults to False, which yields the metadata as
+            a JSON string.
     """
 
     def __init__(
         self,
         events: Iterable[Event],
         namespace: Namespace | None = None,
+        unpack_metadata: bool = False,
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
         self.events = events
+        self.unpack_metadata = unpack_metadata
 
     @classmethod
     def from_dataset(
@@ -36,61 +41,80 @@ class EventsExtractor(BaseExtractor):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
+        unpack_metadata: bool = False,
     ):
-        return cls(cast(Iterable[Event], client.events(data_set_external_ids=data_set_external_id)), namespace)
+        return cls(
+            cast(
+                Iterable[Event],
+                client.events(data_set_external_ids=data_set_external_id),
+            ),
+            namespace,
+            unpack_metadata,
+        )
 
     @classmethod
-    def from_file(cls, file_path: str, namespace: Namespace | None = None):
-        return cls(EventList.load(Path(file_path).read_text()), namespace)
+    def from_file(
+        cls,
+        file_path: str,
+        namespace: Namespace | None = None,
+        unpack_metadata: bool = False,
+    ):
+        return cls(EventList.load(Path(file_path).read_text()), namespace, unpack_metadata)
 
     def extract(self) -> Iterable[Triple]:
         """Extract events as triples."""
         for event in self.events:
-            yield from self._event2triples(event, self.namespace)
+            yield from self._event2triples(event)
 
-    @classmethod
-    def _event2triples(cls, event: Event, namespace: Namespace) -> list[Triple]:
-        id_ = namespace[f"Event_{event.id}"]
+    def _event2triples(self, event: Event) -> list[Triple]:
+        id_ = self.namespace[f"Event_{event.id}"]
 
         # Set rdf type
-        triples: list[Triple] = [(id_, RDF.type, namespace.Event)]
+        triples: list[Triple] = [(id_, RDF.type, self.namespace.Event)]
 
         # Create attributes
 
         if event.external_id:
-            triples.append((id_, namespace.external_id, Literal(event.external_id)))
+            triples.append((id_, self.namespace.external_id, Literal(event.external_id)))
 
         if event.source:
-            triples.append((id_, namespace.type, Literal(event.source)))
+            triples.append((id_, self.namespace.type, Literal(event.source)))
 
         if event.type:
-            triples.append((id_, namespace.type, Literal(event.type)))
+            triples.append((id_, self.namespace.type, Literal(event.type)))
 
         if event.subtype:
-            triples.append((id_, namespace.subtype, Literal(event.subtype)))
+            triples.append((id_, self.namespace.subtype, Literal(event.subtype)))
 
         if event.metadata:
-            for key, value in event.metadata.items():
-                if value:
-                    type_aware_value = string_to_ideal_type(value)
-                    try:
-                        triples.append((id_, namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
-                    except ValidationError:
-                        triples.append((id_, namespace[key], Literal(type_aware_value)))
+            if self.unpack_metadata:
+                for key, value in event.metadata.items():
+                    if value:
+                        type_aware_value = string_to_ideal_type(value)
+                        try:
+                            triples.append((id_, self.namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
+                        except ValidationError:
+                            triples.append((id_, self.namespace[key], Literal(type_aware_value)))
+            else:
+                triples.append((id_, self.namespace.metadata, Literal(json.dumps(event.metadata))))
 
         if event.description:
-            triples.append((id_, namespace.description, Literal(event.description)))
+            triples.append((id_, self.namespace.description, Literal(event.description)))
 
         if event.created_time:
             triples.append(
-                (id_, namespace.created_time, Literal(datetime.fromtimestamp(event.created_time / 1000, timezone.utc)))
+                (
+                    id_,
+                    self.namespace.created_time,
+                    Literal(datetime.fromtimestamp(event.created_time / 1000, timezone.utc)),
+                )
             )
 
         if event.last_updated_time:
             triples.append(
                 (
                     id_,
-                    namespace.last_updated_time,
+                    self.namespace.last_updated_time,
                     Literal(datetime.fromtimestamp(event.last_updated_time / 1000, timezone.utc)),
                 )
             )
@@ -99,7 +123,7 @@ class EventsExtractor(BaseExtractor):
             triples.append(
                 (
                     id_,
-                    namespace.start_time,
+                    self.namespace.start_time,
                     Literal(datetime.fromtimestamp(event.start_time / 1000, timezone.utc)),
                 )
             )
@@ -108,16 +132,22 @@ class EventsExtractor(BaseExtractor):
             triples.append(
                 (
                     id_,
-                    namespace.end_time,
+                    self.namespace.end_time,
                     Literal(datetime.fromtimestamp(event.end_time / 1000, timezone.utc)),
                 )
             )
 
         if event.data_set_id:
-            triples.append((id_, namespace.data_set_id, namespace[f"Dataset_{event.data_set_id}"]))
+            triples.append(
+                (
+                    id_,
+                    self.namespace.data_set_id,
+                    self.namespace[f"Dataset_{event.data_set_id}"],
+                )
+            )
 
         if event.asset_ids:
             for asset_id in event.asset_ids:
-                triples.append((id_, namespace.asset, namespace[f"Asset_{asset_id}"]))
+                triples.append((id_, self.namespace.asset, self.namespace[f"Asset_{asset_id}"]))
 
         return triples

cognite/neat/graph/extractors/_classic_cdf/_files.py

@@ -1,3 +1,4 @@
+import json
 from collections.abc import Iterable
 from datetime import datetime, timezone
 from pathlib import Path
@@ -21,15 +22,19 @@ class FilesExtractor(BaseExtractor):
     Args:
         files_metadata (Iterable[FileMetadata]): An iterable of files metadata.
         namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
+        unpack_metadata (bool, optional): Whether to unpack metadata. Defaults to False, which yields the metadata as
+            a JSON string.
     """
 
     def __init__(
         self,
         files_metadata: Iterable[FileMetadata],
         namespace: Namespace | None = None,
+        unpack_metadata: bool = False,
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
         self.files_metadata = files_metadata
+        self.unpack_metadata = unpack_metadata
 
     @classmethod
     def from_dataset(
@@ -37,56 +42,75 @@ class FilesExtractor(BaseExtractor):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
+        unpack_metadata: bool = False,
     ):
-        return cls(cast(Iterable[FileMetadata], client.files(data_set_external_ids=data_set_external_id)), namespace)
+        return cls(
+            cast(
+                Iterable[FileMetadata],
+                client.files(data_set_external_ids=data_set_external_id),
+            ),
+            namespace,
+            unpack_metadata,
+        )
 
     @classmethod
-    def from_file(cls, file_path: str, namespace: Namespace | None = None):
-        return cls(FileMetadataList.load(Path(file_path).read_text()), namespace)
+    def from_file(
+        cls,
+        file_path: str,
+        namespace: Namespace | None = None,
+        unpack_metadata: bool = False,
+    ):
+        return cls(
+            FileMetadataList.load(Path(file_path).read_text()),
+            namespace,
+            unpack_metadata,
+        )
 
     def extract(self) -> Iterable[Triple]:
         """Extract files metadata as triples."""
         for event in self.files_metadata:
-            yield from self._file2triples(event, self.namespace)
+            yield from self._file2triples(event)
 
-    @classmethod
-    def _file2triples(cls, file: FileMetadata, namespace: Namespace) -> list[Triple]:
-        id_ = namespace[f"File_{file.id}"]
+    def _file2triples(self, file: FileMetadata) -> list[Triple]:
+        id_ = self.namespace[f"File_{file.id}"]
 
         # Set rdf type
-        triples: list[Triple] = [(id_, RDF.type, namespace.File)]
+        triples: list[Triple] = [(id_, RDF.type, self.namespace.File)]
 
         # Create attributes
 
         if file.external_id:
-            triples.append((id_, namespace.external_id, Literal(file.external_id)))
+            triples.append((id_, self.namespace.external_id, Literal(file.external_id)))
 
         if file.source:
-            triples.append((id_, namespace.type, Literal(file.source)))
+            triples.append((id_, self.namespace.type, Literal(file.source)))
 
         if file.mime_type:
-            triples.append((id_, namespace.mime_type, Literal(file.mime_type)))
+            triples.append((id_, self.namespace.mime_type, Literal(file.mime_type)))
 
         if file.uploaded:
-            triples.append((id_, namespace.uploaded, Literal(file.uploaded)))
+            triples.append((id_, self.namespace.uploaded, Literal(file.uploaded)))
 
         if file.source:
-            triples.append((id_, namespace.source, Literal(file.source)))
+            triples.append((id_, self.namespace.source, Literal(file.source)))
 
         if file.metadata:
-            for key, value in file.metadata.items():
-                if value:
-                    type_aware_value = string_to_ideal_type(value)
-                    try:
-                        triples.append((id_, namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
-                    except ValidationError:
-                        triples.append((id_, namespace[key], Literal(type_aware_value)))
+            if self.unpack_metadata:
+                for key, value in file.metadata.items():
+                    if value:
+                        type_aware_value = string_to_ideal_type(value)
+                        try:
+                            triples.append((id_, self.namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
+                        except ValidationError:
+                            triples.append((id_, self.namespace[key], Literal(type_aware_value)))
+            else:
+                triples.append((id_, self.namespace.metadata, Literal(json.dumps(file.metadata))))
 
         if file.source_created_time:
             triples.append(
                 (
                     id_,
-                    namespace.source_created_time,
+                    self.namespace.source_created_time,
                     Literal(datetime.fromtimestamp(file.source_created_time / 1000, timezone.utc)),
                 )
             )
@@ -94,25 +118,33 @@ class FilesExtractor(BaseExtractor):
             triples.append(
                 (
                     id_,
-                    namespace.source_created_time,
+                    self.namespace.source_created_time,
                     Literal(datetime.fromtimestamp(file.source_modified_time / 1000, timezone.utc)),
                 )
             )
         if file.uploaded_time:
             triples.append(
-                (id_, namespace.uploaded_time, Literal(datetime.fromtimestamp(file.uploaded_time / 1000, timezone.utc)))
+                (
+                    id_,
+                    self.namespace.uploaded_time,
+                    Literal(datetime.fromtimestamp(file.uploaded_time / 1000, timezone.utc)),
+                )
             )
 
         if file.created_time:
             triples.append(
-                (id_, namespace.created_time, Literal(datetime.fromtimestamp(file.created_time / 1000, timezone.utc)))
+                (
+                    id_,
+                    self.namespace.created_time,
+                    Literal(datetime.fromtimestamp(file.created_time / 1000, timezone.utc)),
+                )
             )
 
         if file.last_updated_time:
             triples.append(
                 (
                     id_,
-                    namespace.last_updated_time,
+                    self.namespace.last_updated_time,
                     Literal(datetime.fromtimestamp(file.last_updated_time / 1000, timezone.utc)),
                 )
             )
@@ -121,17 +153,29 @@ class FilesExtractor(BaseExtractor):
             for label in file.labels:
                 # external_id can create ill-formed URIs, so we create websafe URIs
                 # since labels do not have internal ids, we use the external_id as the id
-                triples.append((id_, namespace.label, namespace[f"Label_{quote(label.dump()['externalId'])}"]))
+                triples.append(
+                    (
+                        id_,
+                        self.namespace.label,
+                        self.namespace[f"Label_{quote(label.dump()['externalId'])}"],
+                    )
+                )
 
         if file.security_categories:
             for category in file.security_categories:
-                triples.append((id_, namespace.security_categories, Literal(category)))
+                triples.append((id_, self.namespace.security_categories, Literal(category)))
 
         if file.data_set_id:
-            triples.append((id_, namespace.data_set_id, namespace[f"Dataset_{file.data_set_id}"]))
+            triples.append(
+                (
+                    id_,
+                    self.namespace.data_set_id,
+                    self.namespace[f"Dataset_{file.data_set_id}"],
+                )
+            )
 
         if file.asset_ids:
             for asset_id in file.asset_ids:
-                triples.append((id_, namespace.asset, namespace[f"Asset_{asset_id}"]))
+                triples.append((id_, self.namespace.asset, self.namespace[f"Asset_{asset_id}"]))
 
         return triples