cognite-neat 0.85.7__tar.gz → 0.85.9__tar.gz

{cognite_neat-0.85.7 → cognite_neat-0.85.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.85.7
+Version: 0.85.9
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.85.9/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.85.9"

cognite_neat-0.85.9/cognite/neat/graph/extractors/_classic_cdf/_assets.py

@@ -0,0 +1,223 @@
+import json
+import re
+from collections.abc import Callable, Iterable
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import cast
+
+from cognite.client import CogniteClient
+from cognite.client.data_classes import Asset, AssetFilter, AssetList
+from rdflib import RDF, Literal, Namespace
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import create_sha256_hash, string_to_ideal_type
+
+
+class AssetsExtractor(BaseExtractor):
+    """Extract data from Cognite Data Fusions Assets into Neat.
+
+    Args:
+        assets (Iterable[Asset]): An iterable of assets.
+        namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
+        to_type (Callable[[Asset], str | None], optional): A function to convert an asset to a type. Defaults to None.
+            If None or if the function returns None, the asset will be set to the default type "Asset".
+        total (int, optional): The total number of assets to load. If passed, you will get a progress bar if rich
+            is installed. Defaults to None.
+        limit (int, optional): The maximal number of assets to load. Defaults to None. This is typically used for
+            testing setup of the extractor. For example, if you are extracting 100 000 assets, you might want to
+            limit the extraction to 1000 assets to test the setup.
+        unpack_metadata (bool, optional): Whether to unpack metadata. Defaults to False, which yields the metadata as
+            a JSON string.
+    """
+
+    _SPACE_PATTERN = re.compile(r"\s+")
+
+    def __init__(
+        self,
+        assets: Iterable[Asset],
+        namespace: Namespace | None = None,
+        to_type: Callable[[Asset], str | None] | None = None,
+        total: int | None = None,
+        limit: int | None = None,
+        unpack_metadata: bool = True,
+    ):
+        self.namespace = namespace or DEFAULT_NAMESPACE
+        self.assets = assets
+        self.to_type = to_type
+        self.total = total
+        self.limit = min(limit, total) if limit and total else limit
+        self.unpack_metadata = unpack_metadata
+
+    @classmethod
+    def from_dataset(
+        cls,
+        client: CogniteClient,
+        data_set_external_id: str,
+        namespace: Namespace | None = None,
+        to_type: Callable[[Asset], str | None] | None = None,
+        limit: int | None = None,
+        unpack_metadata: bool = True,
+    ):
+        total = client.assets.aggregate_count(filter=AssetFilter(data_set_ids=[{"externalId": data_set_external_id}]))
+
+        return cls(
+            cast(
+                Iterable[Asset],
+                client.assets(data_set_external_ids=data_set_external_id),
+            ),
+            namespace,
+            to_type,
+            total,
+            limit,
+            unpack_metadata=unpack_metadata,
+        )
+
+    @classmethod
+    def from_hierarchy(
+        cls,
+        client: CogniteClient,
+        root_asset_external_id: str,
+        namespace: Namespace | None = None,
+        to_type: Callable[[Asset], str | None] | None = None,
+        limit: int | None = None,
+        unpack_metadata: bool = True,
+    ):
+        total = client.assets.aggregate_count(
+            filter=AssetFilter(asset_subtree_ids=[{"externalId": root_asset_external_id}])
+        )
+
+        return cls(
+            cast(
+                Iterable[Asset],
+                client.assets(asset_subtree_external_ids=root_asset_external_id),
+            ),
+            namespace,
+            to_type,
+            total,
+            limit,
+            unpack_metadata=unpack_metadata,
+        )
+
+    @classmethod
+    def from_file(
+        cls,
+        file_path: str,
+        namespace: Namespace | None = None,
+        to_type: Callable[[Asset], str] | None = None,
+        limit: int | None = None,
+        unpack_metadata: bool = True,
+    ):
+        return cls(
+            AssetList.load(Path(file_path).read_text()),
+            namespace,
+            to_type,
+            limit,
+            unpack_metadata=unpack_metadata,
+        )
+
+    def extract(self) -> Iterable[Triple]:
+        """Extracts an asset with the given asset_id."""
+        if self.total:
+            try:
+                from rich.progress import track
+            except ModuleNotFoundError:
+                to_iterate = self.assets
+            else:
+                to_iterate = track(
+                    self.assets,
+                    total=self.limit or self.total,
+                    description="Extracting Assets",
+                )
+        else:
+            to_iterate = self.assets
+        for no, asset in enumerate(to_iterate):
+            yield from self._asset2triples(asset)
+            if self.limit and no >= self.limit:
+                break
+
+    def _asset2triples(self, asset: Asset) -> list[Triple]:
+        """Converts an asset to triples."""
+        id_ = self.namespace[f"Asset_{asset.id}"]
+
+        # Set rdf type
+        type_ = "Asset"
+        if self.to_type:
+            type_ = self.to_type(asset) or type_
+        type_ = self._SPACE_PATTERN.sub("_", type_)
+
+        triples: list[Triple] = [(id_, RDF.type, self.namespace[type_])]
+
+        # Create attributes
+        if asset.name:
+            triples.append((id_, self.namespace.name, Literal(asset.name)))
+
+        if asset.description:
+            triples.append((id_, self.namespace.description, Literal(asset.description)))
+
+        if asset.external_id:
+            triples.append((id_, self.namespace.external_id, Literal(asset.external_id)))
+
+        if asset.source:
+            triples.append((id_, self.namespace.source, Literal(asset.source)))
+
+        # properties ref creation and update
+        triples.append(
+            (
+                id_,
+                self.namespace.created_time,
+                Literal(datetime.fromtimestamp(asset.created_time / 1000, timezone.utc)),
+            )
+        )
+        triples.append(
+            (
+                id_,
+                self.namespace.last_updated_time,
+                Literal(datetime.fromtimestamp(asset.last_updated_time / 1000, timezone.utc)),
+            )
+        )
+
+        if asset.labels:
+            for label in asset.labels:
+                # external_id can create ill-formed URIs, so we create websafe URIs
+                # since labels do not have internal ids, we use the external_id as the id
+                triples.append(
+                    (
+                        id_,
+                        self.namespace.label,
+                        self.namespace[f"Label_{create_sha256_hash(label.dump()['externalId'])}"],
+                    )
+                )
+
+        if asset.metadata:
+            if self.unpack_metadata:
+                for key, value in asset.metadata.items():
+                    if value:
+                        triples.append(
+                            (
+                                id_,
+                                self.namespace[key],
+                                Literal(string_to_ideal_type(value)),
+                            )
+                        )
+            else:
+                triples.append((id_, self.namespace.metadata, Literal(json.dumps(asset.metadata))))
+
+        # Create connections:
+        if asset.parent_id:
+            triples.append((id_, self.namespace.parent, self.namespace[f"Asset_{asset.parent_id}"]))
+
+        if asset.root_id:
+            triples.append((id_, self.namespace.root, self.namespace[f"Asset_{asset.root_id}"]))
+
+        if asset.data_set_id:
+            triples.append(
+                (
+                    id_,
+                    self.namespace.dataset,
+                    self.namespace[f"Dataset_{asset.data_set_id}"],
+                )
+            )
+
+        return triples

cognite_neat-0.85.9/cognite/neat/graph/extractors/_classic_cdf/_events.py

@@ -0,0 +1,153 @@
+import json
+from collections.abc import Iterable
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import cast
+
+from cognite.client import CogniteClient
+from cognite.client.data_classes import Event, EventList
+from pydantic import AnyHttpUrl, ValidationError
+from rdflib import RDF, Literal, Namespace, URIRef
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import string_to_ideal_type
+
+
+class EventsExtractor(BaseExtractor):
+    """Extract data from Cognite Data Fusions Events into Neat.
+
+    Args:
+        events (Iterable[Event]): An iterable of events.
+        namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
+        unpack_metadata (bool, optional): Whether to unpack metadata. Defaults to False, which yields the metadata as
+            a JSON string.
+    """
+
+    def __init__(
+        self,
+        events: Iterable[Event],
+        namespace: Namespace | None = None,
+        unpack_metadata: bool = True,
+    ):
+        self.namespace = namespace or DEFAULT_NAMESPACE
+        self.events = events
+        self.unpack_metadata = unpack_metadata
+
+    @classmethod
+    def from_dataset(
+        cls,
+        client: CogniteClient,
+        data_set_external_id: str,
+        namespace: Namespace | None = None,
+        unpack_metadata: bool = True,
+    ):
+        return cls(
+            cast(
+                Iterable[Event],
+                client.events(data_set_external_ids=data_set_external_id),
+            ),
+            namespace,
+            unpack_metadata,
+        )
+
+    @classmethod
+    def from_file(
+        cls,
+        file_path: str,
+        namespace: Namespace | None = None,
+        unpack_metadata: bool = True,
+    ):
+        return cls(EventList.load(Path(file_path).read_text()), namespace, unpack_metadata)
+
+    def extract(self) -> Iterable[Triple]:
+        """Extract events as triples."""
+        for event in self.events:
+            yield from self._event2triples(event)
+
+    def _event2triples(self, event: Event) -> list[Triple]:
+        id_ = self.namespace[f"Event_{event.id}"]
+
+        # Set rdf type
+        triples: list[Triple] = [(id_, RDF.type, self.namespace.Event)]
+
+        # Create attributes
+
+        if event.external_id:
+            triples.append((id_, self.namespace.external_id, Literal(event.external_id)))
+
+        if event.source:
+            triples.append((id_, self.namespace.type, Literal(event.source)))
+
+        if event.type:
+            triples.append((id_, self.namespace.type, Literal(event.type)))
+
+        if event.subtype:
+            triples.append((id_, self.namespace.subtype, Literal(event.subtype)))
+
+        if event.metadata:
+            if self.unpack_metadata:
+                for key, value in event.metadata.items():
+                    if value:
+                        type_aware_value = string_to_ideal_type(value)
+                        try:
+                            triples.append((id_, self.namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
+                        except ValidationError:
+                            triples.append((id_, self.namespace[key], Literal(type_aware_value)))
+            else:
+                triples.append((id_, self.namespace.metadata, Literal(json.dumps(event.metadata))))
+
+        if event.description:
+            triples.append((id_, self.namespace.description, Literal(event.description)))
+
+        if event.created_time:
+            triples.append(
+                (
+                    id_,
+                    self.namespace.created_time,
+                    Literal(datetime.fromtimestamp(event.created_time / 1000, timezone.utc)),
+                )
+            )
+
+        if event.last_updated_time:
+            triples.append(
+                (
+                    id_,
+                    self.namespace.last_updated_time,
+                    Literal(datetime.fromtimestamp(event.last_updated_time / 1000, timezone.utc)),
+                )
+            )
+
+        if event.start_time:
+            triples.append(
+                (
+                    id_,
+                    self.namespace.start_time,
+                    Literal(datetime.fromtimestamp(event.start_time / 1000, timezone.utc)),
+                )
+            )
+
+        if event.end_time:
+            triples.append(
+                (
+                    id_,
+                    self.namespace.end_time,
+                    Literal(datetime.fromtimestamp(event.end_time / 1000, timezone.utc)),
+                )
+            )
+
+        if event.data_set_id:
+            triples.append(
+                (
+                    id_,
+                    self.namespace.data_set_id,
+                    self.namespace[f"Dataset_{event.data_set_id}"],
+                )
+            )
+
+        if event.asset_ids:
+            for asset_id in event.asset_ids:
+                triples.append((id_, self.namespace.asset, self.namespace[f"Asset_{asset_id}"]))
+
+        return triples

cognite_neat-0.85.9/cognite/neat/graph/extractors/_classic_cdf/_files.py

@@ -0,0 +1,181 @@
+import json
+from collections.abc import Iterable
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import cast
+from urllib.parse import quote
+
+from cognite.client import CogniteClient
+from cognite.client.data_classes import FileMetadata, FileMetadataList
+from pydantic import AnyHttpUrl, ValidationError
+from rdflib import RDF, Literal, Namespace, URIRef
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import string_to_ideal_type
+
+
+class FilesExtractor(BaseExtractor):
+    """Extract data from Cognite Data Fusions files metadata into Neat.
+
+    Args:
+        files_metadata (Iterable[FileMetadata]): An iterable of files metadata.
+        namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
+        unpack_metadata (bool, optional): Whether to unpack metadata. Defaults to False, which yields the metadata as
+            a JSON string.
+    """
+
+    def __init__(
+        self,
+        files_metadata: Iterable[FileMetadata],
+        namespace: Namespace | None = None,
+        unpack_metadata: bool = True,
+    ):
+        self.namespace = namespace or DEFAULT_NAMESPACE
+        self.files_metadata = files_metadata
+        self.unpack_metadata = unpack_metadata
+
+    @classmethod
+    def from_dataset(
+        cls,
+        client: CogniteClient,
+        data_set_external_id: str,
+        namespace: Namespace | None = None,
+        unpack_metadata: bool = True,
+    ):
+        return cls(
+            cast(
+                Iterable[FileMetadata],
+                client.files(data_set_external_ids=data_set_external_id),
+            ),
+            namespace,
+            unpack_metadata,
+        )
+
+    @classmethod
+    def from_file(
+        cls,
+        file_path: str,
+        namespace: Namespace | None = None,
+        unpack_metadata: bool = True,
+    ):
+        return cls(
+            FileMetadataList.load(Path(file_path).read_text()),
+            namespace,
+            unpack_metadata,
+        )
+
+    def extract(self) -> Iterable[Triple]:
+        """Extract files metadata as triples."""
+        for event in self.files_metadata:
+            yield from self._file2triples(event)
+
+    def _file2triples(self, file: FileMetadata) -> list[Triple]:
+        id_ = self.namespace[f"File_{file.id}"]
+
+        # Set rdf type
+        triples: list[Triple] = [(id_, RDF.type, self.namespace.File)]
+
+        # Create attributes
+
+        if file.external_id:
+            triples.append((id_, self.namespace.external_id, Literal(file.external_id)))
+
+        if file.source:
+            triples.append((id_, self.namespace.type, Literal(file.source)))
+
+        if file.mime_type:
+            triples.append((id_, self.namespace.mime_type, Literal(file.mime_type)))
+
+        if file.uploaded:
+            triples.append((id_, self.namespace.uploaded, Literal(file.uploaded)))
+
+        if file.source:
+            triples.append((id_, self.namespace.source, Literal(file.source)))
+
+        if file.metadata:
+            if self.unpack_metadata:
+                for key, value in file.metadata.items():
+                    if value:
+                        type_aware_value = string_to_ideal_type(value)
+                        try:
+                            triples.append((id_, self.namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
+                        except ValidationError:
+                            triples.append((id_, self.namespace[key], Literal(type_aware_value)))
+            else:
+                triples.append((id_, self.namespace.metadata, Literal(json.dumps(file.metadata))))
+
+        if file.source_created_time:
+            triples.append(
+                (
+                    id_,
+                    self.namespace.source_created_time,
+                    Literal(datetime.fromtimestamp(file.source_created_time / 1000, timezone.utc)),
+                )
+            )
+        if file.source_modified_time:
+            triples.append(
+                (
+                    id_,
+                    self.namespace.source_created_time,
+                    Literal(datetime.fromtimestamp(file.source_modified_time / 1000, timezone.utc)),
+                )
+            )
+        if file.uploaded_time:
+            triples.append(
+                (
+                    id_,
+                    self.namespace.uploaded_time,
+                    Literal(datetime.fromtimestamp(file.uploaded_time / 1000, timezone.utc)),
+                )
+            )
+
+        if file.created_time:
+            triples.append(
+                (
+                    id_,
+                    self.namespace.created_time,
+                    Literal(datetime.fromtimestamp(file.created_time / 1000, timezone.utc)),
+                )
+            )
+
+        if file.last_updated_time:
+            triples.append(
+                (
+                    id_,
+                    self.namespace.last_updated_time,
+                    Literal(datetime.fromtimestamp(file.last_updated_time / 1000, timezone.utc)),
+                )
+            )
+
+        if file.labels:
+            for label in file.labels:
+                # external_id can create ill-formed URIs, so we create websafe URIs
+                # since labels do not have internal ids, we use the external_id as the id
+                triples.append(
+                    (
+                        id_,
+                        self.namespace.label,
+                        self.namespace[f"Label_{quote(label.dump()['externalId'])}"],
+                    )
+                )
+
+        if file.security_categories:
+            for category in file.security_categories:
+                triples.append((id_, self.namespace.security_categories, Literal(category)))
+
+        if file.data_set_id:
+            triples.append(
+                (
+                    id_,
+                    self.namespace.data_set_id,
+                    self.namespace[f"Dataset_{file.data_set_id}"],
+                )
+            )
+
+        if file.asset_ids:
+            for asset_id in file.asset_ids:
+                triples.append((id_, self.namespace.asset, self.namespace[f"Asset_{asset_id}"]))
+
+        return triples

{cognite_neat-0.85.7 → cognite_neat-0.85.9}/cognite/neat/graph/extractors/_classic_cdf/_labels.py

@@ -37,7 +37,11 @@ class LabelsExtractor(BaseExtractor):
         namespace: Namespace | None = None,
     ):
         return cls(
-            cast(Iterable[LabelDefinition], client.labels(data_set_external_ids=data_set_external_id)), namespace
+            cast(
+                Iterable[LabelDefinition],
+                client.labels(data_set_external_ids=data_set_external_id),
+            ),
+            namespace,
         )
 
     @classmethod
@@ -47,36 +51,41 @@ class LabelsExtractor(BaseExtractor):
     def extract(self) -> Iterable[Triple]:
         """Extract labels as triples."""
         for label in self.labels:
-            yield from self._labels2triples(label, self.namespace)
+            yield from self._labels2triples(label)
 
-    @classmethod
-    def _labels2triples(cls, label: LabelDefinition, namespace: Namespace) -> list[Triple]:
+    def _labels2triples(self, label: LabelDefinition) -> list[Triple]:
         if label.external_id:
-            id_ = namespace[f"Label_{create_sha256_hash(label.external_id)}"]
+            id_ = self.namespace[f"Label_{create_sha256_hash(label.external_id)}"]
 
             # Set rdf type
-            triples: list[Triple] = [(id_, RDF.type, namespace.Label)]
+            triples: list[Triple] = [(id_, RDF.type, self.namespace.Label)]
 
             # Create attributes
-            triples.append((id_, namespace.external_id, Literal(label.external_id)))
+            triples.append((id_, self.namespace.external_id, Literal(label.external_id)))
 
             if label.name:
-                triples.append((id_, namespace.name, Literal(label.name)))
+                triples.append((id_, self.namespace.name, Literal(label.name)))
 
             if label.description:
-                triples.append((id_, namespace.description, Literal(label.description)))
+                triples.append((id_, self.namespace.description, Literal(label.description)))
 
             if label.created_time:
                 triples.append(
                     (
                         id_,
-                        namespace.created_time,
+                        self.namespace.created_time,
                         Literal(datetime.fromtimestamp(label.created_time / 1000, timezone.utc)),
                     )
                 )
 
             if label.data_set_id:
-                triples.append((id_, namespace.data_set_id, namespace[f"Dataset_{label.data_set_id}"]))
+                triples.append(
+                    (
+                        id_,
+                        self.namespace.data_set_id,
+                        self.namespace[f"Dataset_{label.data_set_id}"],
+                    )
+                )
 
             return triples
         return []