cognite-neat 0.78.4__py3-none-any.whl → 0.79.0__py3-none-any.whl

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.78.4"
+__version__ = "0.79.0"

cognite/neat/graph/_tracking/__init__.py

@@ -0,0 +1,4 @@
+from .base import Tracker
+from .log import LogTracker
+
+__all__ = ["Tracker", "LogTracker"]

cognite/neat/graph/_tracking/base.py

@@ -0,0 +1,30 @@
+from abc import ABC, abstractmethod
+from collections.abc import Sequence
+
+from cognite.neat.issues import NeatIssue
+
+
+class Tracker(ABC):
+    def __init__(self, name: str, units: list[str], unit_type: str) -> None:
+        self.name = name
+        self.units = units
+        self.unit_type = unit_type
+
+    @abstractmethod
+    def start(self, unit: str) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def finish(self, unit: str) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def _issue(self, issue: NeatIssue) -> None:
+        raise NotImplementedError()
+
+    def issue(self, issue: NeatIssue | Sequence[NeatIssue]) -> None:
+        if isinstance(issue, NeatIssue):
+            self._issue(issue)
+            return
+        for item in issue:
+            self._issue(item)

cognite/neat/graph/_tracking/log.py

@@ -0,0 +1,27 @@
+from logging import getLogger
+
+from cognite.neat.issues import NeatError, NeatIssue, NeatWarning
+
+from .base import Tracker
+
+
+class LogTracker(Tracker):
+    def __init__(self, name: str, units: list[str], unit_type: str) -> None:
+        super().__init__(name, units, unit_type)
+        self._logger = getLogger(__name__)
+        self._total_units = len(units)
+        self._count = 1
+        self._logger.info(f"Staring {self.name} and will process {len(units)} {unit_type}.")
+
+    def start(self, unit: str) -> None:
+        self._logger.info(f"Starting {unit} {self._count}/{self._total_units}.")
+        self._count += 1
+
+    def finish(self, unit: str) -> None:
+        self._logger.info(f"Finished {unit}.")
+
+    def _issue(self, issue: NeatIssue) -> None:
+        if isinstance(issue, NeatWarning):
+            self._logger.warning(issue)
+        elif isinstance(issue, NeatError):
+            self._logger.error(issue)

cognite/neat/graph/extractors/__init__.py

@@ -1,4 +1,19 @@
-from ._classic_cdf._asset_hierarchy import AssetHierarchyExtractor
+from ._classic_cdf._assets import AssetsExtractor
+from ._classic_cdf._events import EventsExtractor
+from ._classic_cdf._files import FilesExtractor
+from ._classic_cdf._labels import LabelsExtractor
+from ._classic_cdf._relationships import RelationshipsExtractor
+from ._classic_cdf._sequences import SequencesExtractor
+from ._classic_cdf._timeseries import TimeSeriesExtractor
 from ._mock_graph_generator import MockGraphGenerator
 
-__all__ = ["AssetHierarchyExtractor", "MockGraphGenerator"]
+__all__ = [
+    "AssetsExtractor",
+    "MockGraphGenerator",
+    "RelationshipsExtractor",
+    "TimeSeriesExtractor",
+    "SequencesExtractor",
+    "EventsExtractor",
+    "FilesExtractor",
+    "LabelsExtractor",
+]

cognite/neat/graph/extractors/_classic_cdf/{_asset_hierarchy.py → _assets.py}

@@ -2,6 +2,7 @@ from collections.abc import Iterable
 from datetime import datetime
 from pathlib import Path
 from typing import cast
+from urllib.parse import quote
 
 import pytz
 from cognite.client import CogniteClient
@@ -14,7 +15,7 @@ from cognite.neat.graph.models import Triple
 from cognite.neat.utils.utils import string_to_ideal_type
 
 
-class AssetHierarchyExtractor(BaseExtractor):
+class AssetsExtractor(BaseExtractor):
     def __init__(
         self,
         assets: Iterable[Asset],
@@ -48,53 +49,59 @@ class AssetHierarchyExtractor(BaseExtractor):
     @classmethod
     def _asset2triples(cls, asset: Asset, namespace: Namespace) -> list[Triple]:
         """Converts an asset to triples."""
-        triples: list[Triple] = [(namespace[str(asset.id)], RDF.type, namespace["Asset"])]
+        id_ = namespace[str(asset.id)]
 
+        # Set rdf type
+        triples: list[Triple] = [(id_, RDF.type, namespace["Asset"])]
+
+        # Create attributes
         if asset.name:
-            triples.append((namespace[str(asset.id)], namespace["name"], Literal(asset.name)))
+            triples.append((id_, namespace.name, Literal(asset.name)))
 
         if asset.description:
-            triples.append((namespace[str(asset.id)], namespace["description"], Literal(asset.description)))
+            triples.append((id_, namespace.description, Literal(asset.description)))
 
         if asset.external_id:
-            triples.append((namespace[str(asset.id)], namespace["externalId"], Literal(asset.external_id)))
+            triples.append((id_, namespace.external_id, Literal(asset.external_id)))
 
         if asset.source:
-            triples.append((namespace[str(asset.id)], namespace["source"], Literal(asset.source)))
+            triples.append((id_, namespace.source, Literal(asset.source)))
 
         # properties ref creation and update
         triples.append(
             (
-                namespace[str(asset.id)],
-                namespace["createdTime"],
+                id_,
+                namespace.created_time,
                 Literal(datetime.fromtimestamp(asset.created_time / 1000, pytz.utc)),
             )
         )
         triples.append(
             (
-                namespace[str(asset.id)],
-                namespace["lastUpdatedTime"],
+                id_,
+                namespace.last_updated_time,
                 Literal(datetime.fromtimestamp(asset.last_updated_time / 1000, pytz.utc)),
             )
         )
 
-        if asset.parent_id:
-            triples.append((namespace[str(asset.id)], namespace["parent"], namespace[str(asset.parent_id)]))
-
-        if asset.root_id:
-            triples.append((namespace[str(asset.id)], namespace["root"], namespace[str(asset.root_id)]))
-
-        if asset.data_set_id:
-            triples.append((namespace[str(asset.id)], namespace["dataset"], namespace[str(asset.data_set_id)]))
-
         if asset.labels:
             for label in asset.labels:
-                # external_id can create ill-formed URIs, so we opt for Literal instead
-                triples.append((namespace[str(asset.id)], namespace["label"], Literal(label.dump()["externalId"])))
+                # external_id can create ill-formed URIs, so we create websafe URIs
+                # since labels do not have internal ids, we use the external_id as the id
+                triples.append((id_, namespace.label, namespace[quote(label.dump()["externalId"])]))
 
         if asset.metadata:
             for key, value in asset.metadata.items():
                 if value:
-                    triples.append((namespace[str(asset.id)], namespace[key], Literal(string_to_ideal_type(value))))
+                    triples.append((id_, namespace[key], Literal(string_to_ideal_type(value))))
+
+        # Create connections:
+        if asset.parent_id:
+            triples.append((id_, namespace.parent, namespace[str(asset.parent_id)]))
+
+        if asset.root_id:
+            triples.append((id_, namespace.root, namespace[str(asset.root_id)]))
+
+        if asset.data_set_id:
+            triples.append((id_, namespace.dataset, namespace[str(asset.data_set_id)]))
 
         return triples

cognite/neat/graph/extractors/_classic_cdf/_events.py

@@ -0,0 +1,117 @@
+from collections.abc import Iterable
+from datetime import datetime
+from pathlib import Path
+from typing import cast
+
+import pytz
+from cognite.client import CogniteClient
+from cognite.client.data_classes import Event, EventList
+from pydantic import AnyHttpUrl, ValidationError
+from rdflib import RDF, Literal, Namespace, URIRef
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import string_to_ideal_type
+
+
+class EventsExtractor(BaseExtractor):
+    def __init__(
+        self,
+        events: Iterable[Event],
+        namespace: Namespace | None = None,
+    ):
+        self.namespace = namespace or DEFAULT_NAMESPACE
+        self.events = events
+
+    @classmethod
+    def from_dataset(
+        cls,
+        client: CogniteClient,
+        data_set_external_id: str,
+        namespace: Namespace | None = None,
+    ):
+        return cls(cast(Iterable[Event], client.events(data_set_external_ids=data_set_external_id)), namespace)
+
+    @classmethod
+    def from_file(cls, file_path: str, namespace: Namespace | None = None):
+        return cls(EventList.load(Path(file_path).read_text()), namespace)
+
+    def extract(self) -> Iterable[Triple]:
+        """Extract events as triples."""
+        for event in self.events:
+            yield from self._event2triples(event, self.namespace)
+
+    @classmethod
+    def _event2triples(cls, event: Event, namespace: Namespace) -> list[Triple]:
+        id_ = namespace[str(event.id)]
+
+        # Set rdf type
+        triples: list[Triple] = [(id_, RDF.type, namespace.Event)]
+
+        # Create attributes
+
+        if event.external_id:
+            triples.append((id_, namespace.external_id, Literal(event.external_id)))
+
+        if event.source:
+            triples.append((id_, namespace.type, Literal(event.source)))
+
+        if event.type:
+            triples.append((id_, namespace.type, Literal(event.type)))
+
+        if event.subtype:
+            triples.append((id_, namespace.subtype, Literal(event.subtype)))
+
+        if event.metadata:
+            for key, value in event.metadata.items():
+                if value:
+                    type_aware_value = string_to_ideal_type(value)
+                    try:
+                        triples.append((id_, namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
+                    except ValidationError:
+                        triples.append((id_, namespace[key], Literal(type_aware_value)))
+
+        if event.description:
+            triples.append((id_, namespace.description, Literal(event.description)))
+
+        if event.created_time:
+            triples.append(
+                (id_, namespace.created_time, Literal(datetime.fromtimestamp(event.created_time / 1000, pytz.utc)))
+            )
+
+        if event.last_updated_time:
+            triples.append(
+                (
+                    id_,
+                    namespace.last_updated_time,
+                    Literal(datetime.fromtimestamp(event.last_updated_time / 1000, pytz.utc)),
+                )
+            )
+
+        if event.start_time:
+            triples.append(
+                (
+                    id_,
+                    namespace.start_time,
+                    Literal(datetime.fromtimestamp(event.start_time / 1000, pytz.utc)),
+                )
+            )
+
+        if event.end_time:
+            triples.append(
+                (
+                    id_,
+                    namespace.end_time,
+                    Literal(datetime.fromtimestamp(event.end_time / 1000, pytz.utc)),
+                )
+            )
+
+        if event.data_set_id:
+            triples.append((id_, namespace.data_set_id, namespace[str(event.data_set_id)]))
+
+        if event.asset_ids:
+            for asset_id in event.asset_ids:
+                triples.append((id_, namespace.asset, namespace[str(asset_id)]))
+
+        return triples

cognite/neat/graph/extractors/_classic_cdf/_files.py

@@ -0,0 +1,131 @@
+from collections.abc import Iterable
+from datetime import datetime
+from pathlib import Path
+from typing import cast
+from urllib.parse import quote
+
+import pytz
+from cognite.client import CogniteClient
+from cognite.client.data_classes import FileMetadata, FileMetadataList
+from pydantic import AnyHttpUrl, ValidationError
+from rdflib import RDF, Literal, Namespace, URIRef
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import string_to_ideal_type
+
+
+class FilesExtractor(BaseExtractor):
+    def __init__(
+        self,
+        events: Iterable[FileMetadata],
+        namespace: Namespace | None = None,
+    ):
+        self.namespace = namespace or DEFAULT_NAMESPACE
+        self.events = events
+
+    @classmethod
+    def from_dataset(
+        cls,
+        client: CogniteClient,
+        data_set_external_id: str,
+        namespace: Namespace | None = None,
+    ):
+        return cls(cast(Iterable[FileMetadata], client.files(data_set_external_ids=data_set_external_id)), namespace)
+
+    @classmethod
+    def from_file(cls, file_path: str, namespace: Namespace | None = None):
+        return cls(FileMetadataList.load(Path(file_path).read_text()), namespace)
+
+    def extract(self) -> Iterable[Triple]:
+        """Extract files metadata as triples."""
+        for event in self.events:
+            yield from self._file2triples(event, self.namespace)
+
+    @classmethod
+    def _file2triples(cls, file: FileMetadata, namespace: Namespace) -> list[Triple]:
+        id_ = namespace[str(file.id)]
+
+        # Set rdf type
+        triples: list[Triple] = [(id_, RDF.type, namespace.File)]
+
+        # Create attributes
+
+        if file.external_id:
+            triples.append((id_, namespace.external_id, Literal(file.external_id)))
+
+        if file.source:
+            triples.append((id_, namespace.type, Literal(file.source)))
+
+        if file.mime_type:
+            triples.append((id_, namespace.mime_type, Literal(file.mime_type)))
+
+        if file.uploaded:
+            triples.append((id_, namespace.uploaded, Literal(file.uploaded)))
+
+        if file.source:
+            triples.append((id_, namespace.source, Literal(file.source)))
+
+        if file.metadata:
+            for key, value in file.metadata.items():
+                if value:
+                    type_aware_value = string_to_ideal_type(value)
+                    try:
+                        triples.append((id_, namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
+                    except ValidationError:
+                        triples.append((id_, namespace[key], Literal(type_aware_value)))
+
+        if file.source_created_time:
+            triples.append(
+                (
+                    id_,
+                    namespace.source_created_time,
+                    Literal(datetime.fromtimestamp(file.source_created_time / 1000, pytz.utc)),
+                )
+            )
+        if file.source_modified_time:
+            triples.append(
+                (
+                    id_,
+                    namespace.source_created_time,
+                    Literal(datetime.fromtimestamp(file.source_modified_time / 1000, pytz.utc)),
+                )
+            )
+        if file.uploaded_time:
+            triples.append(
+                (id_, namespace.uploaded_time, Literal(datetime.fromtimestamp(file.uploaded_time / 1000, pytz.utc)))
+            )
+
+        if file.created_time:
+            triples.append(
+                (id_, namespace.created_time, Literal(datetime.fromtimestamp(file.created_time / 1000, pytz.utc)))
+            )
+
+        if file.last_updated_time:
+            triples.append(
+                (
+                    id_,
+                    namespace.last_updated_time,
+                    Literal(datetime.fromtimestamp(file.last_updated_time / 1000, pytz.utc)),
+                )
+            )
+
+        if file.labels:
+            for label in file.labels:
+                # external_id can create ill-formed URIs, so we create websafe URIs
+                # since labels do not have internal ids, we use the external_id as the id
+                triples.append((id_, namespace.label, namespace[quote(label.dump()["externalId"])]))
+
+        if file.security_categories:
+            for category in file.security_categories:
+                triples.append((id_, namespace.security_categories, Literal(category)))
+
+        if file.data_set_id:
+            triples.append((id_, namespace.data_set_id, namespace[str(file.data_set_id)]))
+
+        if file.asset_ids:
+            for asset_id in file.asset_ids:
+                triples.append((id_, namespace.asset, namespace[str(asset_id)]))
+
+        return triples

cognite/neat/graph/extractors/_classic_cdf/_labels.py

@@ -0,0 +1,72 @@
+from collections.abc import Iterable
+from datetime import datetime
+from pathlib import Path
+from typing import cast
+from urllib.parse import quote
+
+import pytz
+from cognite.client import CogniteClient
+from cognite.client.data_classes import LabelDefinition, LabelDefinitionList
+from rdflib import RDF, Literal, Namespace
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+
+
+class LabelsExtractor(BaseExtractor):
+    def __init__(
+        self,
+        labels: Iterable[LabelDefinition],
+        namespace: Namespace | None = None,
+    ):
+        self.namespace = namespace or DEFAULT_NAMESPACE
+        self.labels = labels
+
+    @classmethod
+    def from_dataset(
+        cls,
+        client: CogniteClient,
+        data_set_external_id: str,
+        namespace: Namespace | None = None,
+    ):
+        return cls(
+            cast(Iterable[LabelDefinition], client.labels(data_set_external_ids=data_set_external_id)), namespace
+        )
+
+    @classmethod
+    def from_file(cls, file_path: str, namespace: Namespace | None = None):
+        return cls(LabelDefinitionList.load(Path(file_path).read_text()), namespace)
+
+    def extract(self) -> Iterable[Triple]:
+        """Extract labels as triples."""
+        for label in self.labels:
+            yield from self._labels2triples(label, self.namespace)
+
+    @classmethod
+    def _labels2triples(cls, label: LabelDefinition, namespace: Namespace) -> list[Triple]:
+        id_ = namespace[quote(cast(str, label.external_id))]
+
+        # Set rdf type
+        triples: list[Triple] = [(id_, RDF.type, namespace.Label)]
+
+        # Create attributes
+
+        if label.external_id:
+            triples.append((id_, namespace.external_id, Literal(label.external_id)))
+
+        if label.name:
+            triples.append((id_, namespace.name, Literal(label.name)))
+
+        if label.description:
+            triples.append((id_, namespace.description, Literal(label.description)))
+
+        if label.created_time:
+            triples.append(
+                (id_, namespace.created_time, Literal(datetime.fromtimestamp(label.created_time / 1000, pytz.utc)))
+            )
+
+        if label.data_set_id:
+            triples.append((id_, namespace.data_set_id, namespace[str(label.data_set_id)]))
+
+        return triples

cognite/neat/graph/extractors/_classic_cdf/_relationships.py

@@ -0,0 +1,153 @@
+import uuid
+from collections.abc import Iterable
+from datetime import datetime
+from pathlib import Path
+from typing import cast
+from urllib.parse import quote
+
+import pytz
+from cognite.client import CogniteClient
+from cognite.client.data_classes import Relationship, RelationshipList
+from rdflib import RDF, Literal, Namespace
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+
+
+class RelationshipsExtractor(BaseExtractor):
+    def __init__(
+        self,
+        relationships: Iterable[Relationship],
+        namespace: Namespace | None = None,
+    ):
+        self.namespace = namespace or DEFAULT_NAMESPACE
+        self.relationships = relationships
+
+    @classmethod
+    def from_dataset(
+        cls,
+        client: CogniteClient,
+        data_set_external_id: str,
+        namespace: Namespace | None = None,
+    ):
+        return cls(
+            cast(Iterable[Relationship], client.relationships(data_set_external_ids=data_set_external_id)), namespace
+        )
+
+    @classmethod
+    def from_file(cls, file_path: str, namespace: Namespace | None = None):
+        return cls(RelationshipList.load(Path(file_path).read_text()), namespace)
+
+    def extract(self) -> Iterable[Triple]:
+        """Extracts an asset with the given asset_id."""
+        for relationship in self.relationships:
+            yield from self._relationship2triples(relationship, self.namespace)
+
+    @classmethod
+    def _relationship2triples(cls, relationship: Relationship, namespace: Namespace) -> list[Triple]:
+        """Converts an asset to triples."""
+
+        # relationships do not have an internal id, so we generate one
+        id_ = namespace[str(uuid.uuid4())]
+
+        # Set rdf type
+        triples: list[Triple] = [(id_, RDF.type, namespace["Relationship"])]
+
+        # Set source and target types
+        if relationship.source_type:
+            triples.append(
+                (
+                    id_,
+                    namespace.source_type,
+                    namespace[relationship.source_type.title()],
+                )
+            )
+
+        if relationship.target_type:
+            triples.append(
+                (
+                    id_,
+                    namespace.target_type,
+                    namespace[relationship.target_type.title()],
+                )
+            )
+
+        # Create attributes
+        if relationship.external_id:
+            triples.append((id_, namespace.external_id, Literal(relationship.external_id)))
+
+        if relationship.source_external_id:
+            triples.append(
+                (
+                    id_,
+                    namespace.source_external_id,
+                    Literal(relationship.source_external_id),
+                )
+            )
+
+        if relationship.target_external_id:
+            triples.append(
+                (
+                    id_,
+                    namespace.target_external_id,
+                    Literal(relationship.target_external_id),
+                )
+            )
+
+        if relationship.start_time:
+            triples.append(
+                (
+                    id_,
+                    namespace.start_time,
+                    Literal(datetime.fromtimestamp(relationship.start_time / 1000, pytz.utc)),
+                )
+            )
+
+        if relationship.end_time:
+            triples.append(
+                (
+                    id_,
+                    namespace.end_time,
+                    Literal(datetime.fromtimestamp(relationship.end_time / 1000, pytz.utc)),
+                )
+            )
+
+        if relationship.created_time:
+            triples.append(
+                (
+                    id_,
+                    namespace.created_time,
+                    Literal(datetime.fromtimestamp(relationship.created_time / 1000, pytz.utc)),
+                )
+            )
+
+        if relationship.last_updated_time:
+            triples.append(
+                (
+                    id_,
+                    namespace.last_updated_time,
+                    Literal(datetime.fromtimestamp(relationship.last_updated_time / 1000, pytz.utc)),
+                )
+            )
+
+        if relationship.confidence:
+            triples.append(
+                (
+                    id_,
+                    namespace.confidence,
+                    Literal(relationship.confidence),
+                )
+            )
+
+        if relationship.labels:
+            for label in relationship.labels:
+                # external_id can create ill-formed URIs, so we create websafe URIs
+                # since labels do not have internal ids, we use the external_id as the id
+                triples.append((id_, namespace.label, namespace[quote(label.dump()["externalId"])]))
+
+        # Create connection
+        if relationship.data_set_id:
+            triples.append((id_, namespace.dataset, namespace[str(relationship.data_set_id)]))
+
+        return triples