cognite-neat 0.78.3__py3-none-any.whl → 0.78.5__py3-none-any.whl

cognite/neat/graph/extractors/_classic_cdf/_sequences.py

@@ -0,0 +1,92 @@
+from collections.abc import Iterable
+from datetime import datetime
+from pathlib import Path
+from typing import cast
+
+import pytz
+from cognite.client import CogniteClient
+from cognite.client.data_classes import Sequence, SequenceList
+from pydantic import AnyHttpUrl, ValidationError
+from rdflib import RDF, Literal, Namespace, URIRef
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import string_to_ideal_type
+
+
+class SequencesExtractor(BaseExtractor):
+    def __init__(
+        self,
+        sequence: Iterable[Sequence],
+        namespace: Namespace | None = None,
+    ):
+        self.namespace = namespace or DEFAULT_NAMESPACE
+        self.sequence = sequence
+
+    @classmethod
+    def from_dataset(
+        cls,
+        client: CogniteClient,
+        data_set_external_id: str,
+        namespace: Namespace | None = None,
+    ):
+        return cls(cast(Iterable[Sequence], client.sequences(data_set_external_ids=data_set_external_id)), namespace)
+
+    @classmethod
+    def from_file(cls, file_path: str, namespace: Namespace | None = None):
+        return cls(SequenceList.load(Path(file_path).read_text()), namespace)
+
+    def extract(self) -> Iterable[Triple]:
+        """Extract sequences as triples."""
+        for sequence in self.sequence:
+            yield from self._sequence2triples(sequence, self.namespace)
+
+    @classmethod
+    def _sequence2triples(cls, sequence: Sequence, namespace: Namespace) -> list[Triple]:
+        id_ = namespace[str(sequence.id)]
+
+        # Set rdf type
+        triples: list[Triple] = [(id_, RDF.type, namespace.Sequence)]
+
+        # Create attributes
+
+        if sequence.external_id:
+            triples.append((id_, namespace.external_id, Literal(sequence.external_id)))
+
+        if sequence.name:
+            triples.append((id_, namespace.name, Literal(sequence.name)))
+
+        if sequence.metadata:
+            for key, value in sequence.metadata.items():
+                if value:
+                    type_aware_value = string_to_ideal_type(value)
+                    try:
+                        triples.append((id_, namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
+                    except ValidationError:
+                        triples.append((id_, namespace[key], Literal(type_aware_value)))
+
+        if sequence.description:
+            triples.append((id_, namespace.description, Literal(sequence.description)))
+
+        if sequence.created_time:
+            triples.append(
+                (id_, namespace.created_time, Literal(datetime.fromtimestamp(sequence.created_time / 1000, pytz.utc)))
+            )
+
+        if sequence.last_updated_time:
+            triples.append(
+                (
+                    id_,
+                    namespace.last_updated_time,
+                    Literal(datetime.fromtimestamp(sequence.last_updated_time / 1000, pytz.utc)),
+                )
+            )
+
+        if sequence.data_set_id:
+            triples.append((id_, namespace.data_set_id, namespace[str(sequence.data_set_id)]))
+
+        if sequence.asset_id:
+            triples.append((id_, namespace.asset, namespace[str(sequence.asset_id)]))
+
+        return triples

cognite/neat/graph/extractors/_classic_cdf/_timeseries.py

@@ -0,0 +1,118 @@
+from collections.abc import Iterable
+from datetime import datetime
+from pathlib import Path
+from typing import cast
+
+import pytz
+from cognite.client import CogniteClient
+from cognite.client.data_classes import TimeSeries, TimeSeriesList
+from pydantic import AnyHttpUrl, ValidationError
+from rdflib import RDF, Literal, Namespace, URIRef
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import string_to_ideal_type
+
+
+class TimeSeriesExtractor(BaseExtractor):
+    def __init__(
+        self,
+        timeseries: Iterable[TimeSeries],
+        namespace: Namespace | None = None,
+    ):
+        self.namespace = namespace or DEFAULT_NAMESPACE
+        self.timeseries = timeseries
+
+    @classmethod
+    def from_dataset(
+        cls,
+        client: CogniteClient,
+        data_set_external_id: str,
+        namespace: Namespace | None = None,
+    ):
+        return cls(
+            cast(Iterable[TimeSeries], client.time_series(data_set_external_ids=data_set_external_id)), namespace
+        )
+
+    @classmethod
+    def from_file(cls, file_path: str, namespace: Namespace | None = None):
+        return cls(TimeSeriesList.load(Path(file_path).read_text()), namespace)
+
+    def extract(self) -> Iterable[Triple]:
+        """Extract timeseries as triples."""
+        for timeseries in self.timeseries:
+            yield from self._timeseries2triples(timeseries, self.namespace)
+
+    @classmethod
+    def _timeseries2triples(cls, timeseries: TimeSeries, namespace: Namespace) -> list[Triple]:
+        id_ = namespace[str(timeseries.id)]
+
+        # Set rdf type
+        triples: list[Triple] = [(id_, RDF.type, namespace.TimeSeries)]
+
+        # Create attributes
+
+        if timeseries.external_id:
+            triples.append((id_, namespace.external_id, Literal(timeseries.external_id)))
+
+        if timeseries.name:
+            triples.append((id_, namespace.name, Literal(timeseries.name)))
+
+        if timeseries.is_string:
+            triples.append((id_, namespace.is_string, Literal(timeseries.is_string)))
+
+        if timeseries.metadata:
+            for key, value in timeseries.metadata.items():
+                if value:
+                    type_aware_value = string_to_ideal_type(value)
+                    try:
+                        triples.append((id_, namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
+                    except ValidationError:
+                        triples.append((id_, namespace[key], Literal(type_aware_value)))
+
+        if timeseries.unit:
+            triples.append((id_, namespace.unit, Literal(timeseries.unit)))
+
+        if namespace.is_step:
+            triples.append((id_, namespace.is_step, Literal(timeseries.is_step)))
+
+        if timeseries.description:
+            triples.append((id_, namespace.description, Literal(timeseries.description)))
+
+        if timeseries.security_categories:
+            for category in timeseries.security_categories:
+                triples.append((id_, namespace.security_categories, Literal(category)))
+
+        if timeseries.created_time:
+            triples.append(
+                (id_, namespace.created_time, Literal(datetime.fromtimestamp(timeseries.created_time / 1000, pytz.utc)))
+            )
+
+        if timeseries.last_updated_time:
+            triples.append(
+                (
+                    id_,
+                    namespace.last_updated_time,
+                    Literal(datetime.fromtimestamp(timeseries.last_updated_time / 1000, pytz.utc)),
+                )
+            )
+
+        if timeseries.legacy_name:
+            triples.append((id_, namespace.legacy_name, Literal(timeseries.legacy_name)))
+
+        # Create connections
+        if timeseries.unit_external_id:
+            # try to create connection to QUDT unit catalog
+            try:
+                triples.append((id_, namespace.unit_external_id, URIRef(str(AnyHttpUrl(timeseries.unit_external_id)))))
+            except ValidationError:
+                triples.append((id_, namespace.unit_external_id, Literal(timeseries.unit_external_id)))
+
+        if timeseries.data_set_id:
+            triples.append((id_, namespace.dataset, namespace[str(timeseries.data_set_id)]))
+
+        if timeseries.asset_id:
+            triples.append((id_, namespace.asset, namespace[str(timeseries.asset_id)]))
+
+        return triples

cognite/neat/graph/issues/loader.py

@@ -0,0 +1,104 @@
+from dataclasses import dataclass
+from typing import Any
+
+from cognite.neat.issues import NeatError, NeatWarning
+
+__all__ = [
+    "FailedAuthorizationError",
+    "MissingDataModelError",
+    "FailedConvertError",
+    "InvalidClassWarning",
+    "InvalidInstanceError",
+]
+
+
+@dataclass(frozen=True)
+class FailedAuthorizationError(NeatError):
+    description = "Missing authorization for {action}: {reason}"
+
+    action: str
+    reason: str
+
+    def message(self) -> str:
+        return self.description.format(action=self.action, reason=self.reason)
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["action"] = self.action
+        output["reason"] = self.reason
+        return output
+
+
+@dataclass(frozen=True)
+class MissingDataModelError(NeatError):
+    description = "The data model with identifier {identifier} is missing: {reason}"
+    fix = "Check the data model identifier and try again."
+
+    identifier: str
+    reason: str
+
+    def message(self) -> str:
+        return self.description.format(identifier=self.identifier, reason=self.reason)
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["identifier"] = self.identifier
+        output["reason"] = self.reason
+        return output
+
+
+@dataclass(frozen=True)
+class FailedConvertError(NeatError):
+    description = "Failed to convert the {identifier} to {target_format}: {reason}"
+    fix = "Check the error message and correct the rules."
+    identifier: str
+    target_format: str
+    reason: str
+
+    def message(self) -> str:
+        return self.description.format(identifier=self.identifier, target_format=self.target_format, reason=self.reason)
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["identifier"] = self.identifier
+        output["targetFormat"] = self.target_format
+        output["reason"] = self.reason
+        return output
+
+
+@dataclass(frozen=True)
+class InvalidClassWarning(NeatWarning):
+    description = "The class {class_name} is invalid and will be skipped. {reason}"
+    fix = "Check the error message and correct the class."
+
+    class_name: str
+    reason: str
+
+    def message(self) -> str:
+        return self.description.format(class_name=self.class_name, reason=self.reason)
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["class_name"] = self.class_name
+        output["reason"] = self.reason
+        return output
+
+
+@dataclass(frozen=True)
+class InvalidInstanceError(NeatError):
+    description = "The {type_} with identifier {identifier} is invalid and will be skipped. {reason}"
+    fix = "Check the error message and correct the instance."
+
+    type_: str
+    identifier: str
+    reason: str
+
+    def message(self) -> str:
+        return self.description.format(type_=self.type_, identifier=self.identifier, reason=self.reason)
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["type"] = self.type_
+        output["identifier"] = self.identifier
+        output["reason"] = self.reason
+        return output

cognite/neat/graph/loaders/__init__.py

@@ -0,0 +1,4 @@
+from ._base import BaseLoader, CDFLoader
+from ._rdf2dms import DMSLoader
+
+__all__ = ["BaseLoader", "CDFLoader", "DMSLoader"]

cognite/neat/graph/loaders/_base.py

@@ -0,0 +1,109 @@
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from pathlib import Path
+from typing import ClassVar, Generic, Literal, TypeVar, overload
+
+from cognite.client import CogniteClient
+from cognite.client.data_classes.capabilities import Capability
+
+from cognite.neat.graph import NeatGraphStoreBase
+from cognite.neat.graph.issues.loader import FailedAuthorizationError
+from cognite.neat.issues import NeatIssue, NeatIssueList
+from cognite.neat.utils.upload import UploadDiffsID, UploadResultIDs
+
+T_Output = TypeVar("T_Output")
+
+
+class BaseLoader(ABC, Generic[T_Output]):
+    _new_line = "\n"
+    _encoding = "utf-8"
+
+    def __init__(self, graph_store: NeatGraphStoreBase):
+        self.graph_store = graph_store
+
+    @abstractmethod
+    def write_to_file(self, filepath: Path) -> None:
+        raise NotImplementedError
+
+    def load(self, stop_on_exception: bool = False) -> Iterable[T_Output | NeatIssue]:
+        """Load the graph with data."""
+        return self._load(stop_on_exception)
+
+    @abstractmethod
+    def _load(self, stop_on_exception: bool = False) -> Iterable[T_Output | NeatIssue]:
+        """Load the graph with data."""
+        pass
+
+
+class CDFLoader(BaseLoader[T_Output]):
+    _UPLOAD_BATCH_SIZE: ClassVar[int] = 1000
+
+    @overload
+    def load_into_cdf_iterable(
+        self, client: CogniteClient, return_diffs: Literal[False] = False, dry_run: bool = False
+    ) -> Iterable[UploadResultIDs]: ...
+
+    @overload
+    def load_into_cdf_iterable(
+        self, client: CogniteClient, return_diffs: Literal[True], dry_run: bool = False
+    ) -> Iterable[UploadDiffsID]: ...
+
+    def load_into_cdf_iterable(
+        self, client: CogniteClient, return_diffs: bool = False, dry_run: bool = False
+    ) -> Iterable[UploadResultIDs] | Iterable[UploadDiffsID]:
+        yield from self._load_into_cdf_iterable(client, return_diffs, dry_run)
+
+    @overload
+    def load_into_cdf(
+        self, client: CogniteClient, return_diffs: Literal[False] = False, dry_run: bool = False
+    ) -> list[UploadResultIDs]: ...
+
+    @overload
+    def load_into_cdf(
+        self, client: CogniteClient, return_diffs: Literal[True], dry_run: bool = False
+    ) -> list[UploadDiffsID]: ...
+
+    def load_into_cdf(
+        self, client: CogniteClient, return_diffs: bool = False, dry_run: bool = False
+    ) -> list[UploadResultIDs] | list[UploadDiffsID]:
+        return list(self._load_into_cdf_iterable(client, return_diffs, dry_run))  # type: ignore[return-value]
+
+    def _load_into_cdf_iterable(
+        self, client: CogniteClient, return_diffs: bool = False, dry_run: bool = False
+    ) -> Iterable[UploadResultIDs] | Iterable[UploadDiffsID]:
+        missing_capabilities = client.iam.verify_capabilities(self._get_required_capabilities())
+        result_cls = UploadDiffsID if return_diffs else UploadResultIDs
+        if missing_capabilities:
+            result = result_cls(name=type(self).__name__)
+            result.issues.append(FailedAuthorizationError(action="Upload to CDF", reason=str(missing_capabilities)))
+            yield result
+            return
+
+        issues = NeatIssueList[NeatIssue]()
+        items: list[T_Output] = []
+        for result in self.load(stop_on_exception=False):
+            if isinstance(result, NeatIssue):
+                issues.append(result)
+            else:
+                items.append(result)
+
+            if len(items) >= self._UPLOAD_BATCH_SIZE:
+                yield self._upload_to_cdf(client, items, return_diffs, dry_run, issues)
+                items.clear()
+        if items:
+            yield self._upload_to_cdf(client, items, return_diffs, dry_run, issues)
+
+    @abstractmethod
+    def _get_required_capabilities(self) -> list[Capability]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def _upload_to_cdf(
+        self,
+        client: CogniteClient,
+        items: list[T_Output],
+        return_diffs: bool,
+        dry_run: bool,
+        read_issues: NeatIssueList,
+    ) -> UploadResultIDs | UploadDiffsID:
+        raise NotImplementedError