cognite-neat 0.81.9__tar.gz → 0.81.11__tar.gz

{cognite_neat-0.81.9 → cognite_neat-0.81.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.81.9
+Version: 0.81.11
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.81.11/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.81.11"

{cognite_neat-0.81.9 → cognite_neat-0.81.11}/cognite/neat/graph/extractors/_classic_cdf/_assets.py

@@ -2,7 +2,6 @@ from collections.abc import Iterable
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import cast
-from urllib.parse import quote
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes import Asset, AssetList
@@ -11,7 +10,7 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.utils import create_sha256_hash, string_to_ideal_type
 
 
 class AssetsExtractor(BaseExtractor):
@@ -86,7 +85,9 @@ class AssetsExtractor(BaseExtractor):
             for label in asset.labels:
                 # external_id can create ill-formed URIs, so we create websafe URIs
                 # since labels do not have internal ids, we use the external_id as the id
-                triples.append((id_, namespace.label, namespace[f"Label_{quote(label.dump()['externalId'])}"]))
+                triples.append(
+                    (id_, namespace.label, namespace[f"Label_{create_sha256_hash(label.dump()['externalId'])}"])
+                )
 
         if asset.metadata:
             for key, value in asset.metadata.items():

{cognite_neat-0.81.9 → cognite_neat-0.81.11}/cognite/neat/graph/extractors/_classic_cdf/_labels.py

@@ -2,7 +2,6 @@ from collections.abc import Iterable
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import cast
-from urllib.parse import quote
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes import LabelDefinition, LabelDefinitionList
@@ -11,6 +10,7 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import create_sha256_hash
 
 
 class LabelsExtractor(BaseExtractor):
@@ -44,28 +44,32 @@ class LabelsExtractor(BaseExtractor):
 
     @classmethod
     def _labels2triples(cls, label: LabelDefinition, namespace: Namespace) -> list[Triple]:
-        id_ = namespace[f"Label_{quote(label.dump()['externalId'])}"]
-
-        # Set rdf type
-        triples: list[Triple] = [(id_, RDF.type, namespace.Label)]
+        if label.external_id:
+            id_ = namespace[f"Label_{create_sha256_hash(label.external_id)}"]
 
-        # Create attributes
+            # Set rdf type
+            triples: list[Triple] = [(id_, RDF.type, namespace.Label)]
 
-        if label.external_id:
+            # Create attributes
             triples.append((id_, namespace.external_id, Literal(label.external_id)))
 
-        if label.name:
-            triples.append((id_, namespace.name, Literal(label.name)))
+            if label.name:
+                triples.append((id_, namespace.name, Literal(label.name)))
 
-        if label.description:
-            triples.append((id_, namespace.description, Literal(label.description)))
+            if label.description:
+                triples.append((id_, namespace.description, Literal(label.description)))
 
-        if label.created_time:
-            triples.append(
-                (id_, namespace.created_time, Literal(datetime.fromtimestamp(label.created_time / 1000, timezone.utc)))
-            )
+            if label.created_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.created_time,
+                        Literal(datetime.fromtimestamp(label.created_time / 1000, timezone.utc)),
+                    )
+                )
 
-        if label.data_set_id:
-            triples.append((id_, namespace.data_set_id, namespace[f"Dataset_{label.data_set_id}"]))
+            if label.data_set_id:
+                triples.append((id_, namespace.data_set_id, namespace[f"Dataset_{label.data_set_id}"]))
 
-        return triples
+            return triples
+        return []

cognite_neat-0.81.11/cognite/neat/graph/extractors/_classic_cdf/_relationships.py

@@ -0,0 +1,152 @@
+from collections.abc import Iterable
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import cast
+from urllib.parse import quote
+
+from cognite.client import CogniteClient
+from cognite.client.data_classes import Relationship, RelationshipList
+from rdflib import RDF, Literal, Namespace
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import create_sha256_hash
+
+
+class RelationshipsExtractor(BaseExtractor):
+    def __init__(
+        self,
+        relationships: Iterable[Relationship],
+        namespace: Namespace | None = None,
+    ):
+        self.namespace = namespace or DEFAULT_NAMESPACE
+        self.relationships = relationships
+
+    @classmethod
+    def from_dataset(
+        cls,
+        client: CogniteClient,
+        data_set_external_id: str,
+        namespace: Namespace | None = None,
+    ):
+        return cls(
+            cast(Iterable[Relationship], client.relationships(data_set_external_ids=data_set_external_id)), namespace
+        )
+
+    @classmethod
+    def from_file(cls, file_path: str, namespace: Namespace | None = None):
+        return cls(RelationshipList.load(Path(file_path).read_text()), namespace)
+
+    def extract(self) -> Iterable[Triple]:
+        """Extracts an asset with the given asset_id."""
+        for relationship in self.relationships:
+            yield from self._relationship2triples(relationship, self.namespace)
+
+    @classmethod
+    def _relationship2triples(cls, relationship: Relationship, namespace: Namespace) -> list[Triple]:
+        """Converts an asset to triples."""
+
+        if relationship.external_id and relationship.source_external_id and relationship.target_external_id:
+            # relationships do not have an internal id, so we generate one
+            id_ = namespace[f"Relationship_{create_sha256_hash(relationship.external_id)}"]
+
+            # Set rdf type
+            triples: list[Triple] = [(id_, RDF.type, namespace["Relationship"])]
+
+            # Set source and target types
+            if source_type := relationship.source_type:
+                triples.append(
+                    (
+                        id_,
+                        namespace.source_type,
+                        namespace[source_type.title()],
+                    )
+                )
+
+            if target_type := relationship.target_type:
+                triples.append(
+                    (
+                        id_,
+                        namespace.target_type,
+                        namespace[target_type.title()],
+                    )
+                )
+
+            # Create attributes
+
+            triples.append((id_, namespace.external_id, Literal(relationship.external_id)))
+
+            triples.append(
+                (
+                    id_,
+                    namespace.source_external_id,
+                    Literal(relationship.source_external_id),
+                )
+            )
+
+            triples.append(
+                (
+                    id_,
+                    namespace.target_external_id,
+                    Literal(relationship.target_external_id),
+                )
+            )
+
+            if relationship.start_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.start_time,
+                        Literal(datetime.fromtimestamp(relationship.start_time / 1000, timezone.utc)),
+                    )
+                )
+
+            if relationship.end_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.end_time,
+                        Literal(datetime.fromtimestamp(relationship.end_time / 1000, timezone.utc)),
+                    )
+                )
+
+            if relationship.created_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.created_time,
+                        Literal(datetime.fromtimestamp(relationship.created_time / 1000, timezone.utc)),
+                    )
+                )
+
+            if relationship.last_updated_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.last_updated_time,
+                        Literal(datetime.fromtimestamp(relationship.last_updated_time / 1000, timezone.utc)),
+                    )
+                )
+
+            if relationship.confidence:
+                triples.append(
+                    (
+                        id_,
+                        namespace.confidence,
+                        Literal(relationship.confidence),
+                    )
+                )
+
+            if relationship.labels:
+                for label in relationship.labels:
+                    # external_id can create ill-formed URIs, so we create websafe URIs
+                    # since labels do not have internal ids, we use the external_id as the id
+                    triples.append((id_, namespace.label, namespace[f"Label_{quote(label.dump()['externalId'])}"]))
+
+            # Create connection
+            if relationship.data_set_id:
+                triples.append((id_, namespace.dataset, namespace[f"Dataset_{relationship.data_set_id}"]))
+
+            return triples
+        return []

cognite_neat-0.81.11/cognite/neat/graph/transformers/__init__.py

@@ -0,0 +1,26 @@
+from ._classic_cdf import (
+    AddAssetDepth,
+    AssetEventConnector,
+    AssetFileConnector,
+    AssetRelationshipConnector,
+    AssetSequenceConnector,
+    AssetTimeSeriesConnector,
+)
+
+__all__ = [
+    "AddAssetDepth",
+    "AssetTimeSeriesConnector",
+    "AssetSequenceConnector",
+    "AssetFileConnector",
+    "AssetEventConnector",
+    "AssetRelationshipConnector",
+]
+
+Transformers = (
+    AddAssetDepth
+    | AssetTimeSeriesConnector
+    | AssetSequenceConnector
+    | AssetFileConnector
+    | AssetEventConnector
+    | AssetRelationshipConnector
+)

{cognite_neat-0.81.9 → cognite_neat-0.81.11}/cognite/neat/graph/transformers/_classic_cdf.py

@@ -169,3 +169,100 @@ class AssetFileConnector(BaseTransformer):
                 # files can be connected to multiple assets in the graph
                 for (asset_id,) in cast(list[tuple], assets_id_res):
                     graph.add((asset_id, DEFAULT_NAMESPACE.file, file_id))
+
+
+class AssetEventConnector(BaseTransformer):
+    description: str = "Connects assets to events, thus forming bi-directional connection"
+    _use_only_once: bool = True
+    _need_changes = frozenset({str(extractors.AssetsExtractor.__name__), str(extractors.EventsExtractor.__name__)})
+    _asset_template: str = """SELECT ?asset_id WHERE {{
+                              <{event_id}> <{asset_prop}> ?asset_id .
+                              ?asset_id a <{asset_type}>}}"""
+
+    def __init__(
+        self,
+        asset_type: URIRef | None = None,
+        event_type: URIRef | None = None,
+        asset_prop: URIRef | None = None,
+    ):
+        self.asset_type = asset_type or DEFAULT_NAMESPACE.Asset
+        self.event_type = event_type or DEFAULT_NAMESPACE.Event
+        self.asset_prop = asset_prop or DEFAULT_NAMESPACE.asset
+
+    def transform(self, graph: Graph) -> None:
+        for event_id_result in graph.query(f"SELECT DISTINCT ?event_id WHERE {{?event_id a <{self.event_type}>}}"):
+            event_id: URIRef = cast(tuple, event_id_result)[0]
+
+            if assets_id_res := list(
+                graph.query(
+                    self._asset_template.format(
+                        event_id=event_id,
+                        asset_prop=self.asset_prop,
+                        asset_type=self.asset_type,
+                    )
+                )
+            ):
+                # files can be connected to multiple assets in the graph
+                for (asset_id,) in cast(list[tuple], assets_id_res):
+                    graph.add((asset_id, DEFAULT_NAMESPACE.event, event_id))
+
+
+class AssetRelationshipConnector(BaseTransformer):
+    description: str = "Connects assets via relationships"
+    _use_only_once: bool = True
+    _need_changes = frozenset(
+        {str(extractors.AssetsExtractor.__name__), str(extractors.RelationshipsExtractor.__name__)}
+    )
+    _asset_template: str = """SELECT ?source ?target WHERE {{
+                              <{relationship_id}> <{relationship_source_xid_prop}> ?source_xid .
+                              ?source <{asset_xid_property}> ?source_xid .
+                              ?source a <{asset_type}> .
+
+                              <{relationship_id}> <{relationship_target_xid_prop}> ?target_xid .
+                              ?target <{asset_xid_property}> ?target_xid .
+                              ?target a <{asset_type}> .}}"""
+
+    def __init__(
+        self,
+        asset_type: URIRef | None = None,
+        relationship_type: URIRef | None = None,
+        relationship_source_xid_prop: URIRef | None = None,
+        relationship_target_xid_prop: URIRef | None = None,
+        asset_xid_property: URIRef | None = None,
+    ):
+        self.asset_type = asset_type or DEFAULT_NAMESPACE.Asset
+        self.relationship_type = relationship_type or DEFAULT_NAMESPACE.Relationship
+        self.relationship_source_xid_prop = relationship_source_xid_prop or DEFAULT_NAMESPACE.source_external_id
+        self.relationship_target_xid_prop = relationship_target_xid_prop or DEFAULT_NAMESPACE.target_external_id
+        self.asset_xid_property = asset_xid_property or DEFAULT_NAMESPACE.external_id
+
+    def transform(self, graph: Graph) -> None:
+        for relationship_id_result in graph.query(
+            f"SELECT DISTINCT ?relationship_id WHERE {{?relationship_id a <{self.relationship_type}>}}"
+        ):
+            relationship_id: URIRef = cast(tuple, relationship_id_result)[0]
+
+            if assets_id_res := list(
+                graph.query(
+                    self._asset_template.format(
+                        relationship_id=relationship_id,
+                        asset_xid_property=self.asset_xid_property,
+                        relationship_source_xid_prop=self.relationship_source_xid_prop,
+                        relationship_target_xid_prop=self.relationship_target_xid_prop,
+                        asset_type=self.asset_type,
+                    )
+                )
+            ):
+                # files can be connected to multiple assets in the graph
+                for source_asset_id, target_asset_id in cast(list[tuple], assets_id_res):
+                    # create a relationship between the two assets
+                    graph.add((source_asset_id, DEFAULT_NAMESPACE.relationship, relationship_id))
+                    graph.add((target_asset_id, DEFAULT_NAMESPACE.relationship, relationship_id))
+
+                    # add source and target to the relationship
+                    graph.add((relationship_id, DEFAULT_NAMESPACE.source, source_asset_id))
+                    graph.add((relationship_id, DEFAULT_NAMESPACE.target, target_asset_id))
+
+                    # remove properties that are not needed, specifically the external ids
+                    graph.remove((relationship_id, self.relationship_source_xid_prop, None))
+                    graph.remove((relationship_id, self.relationship_target_xid_prop, None))

{cognite_neat-0.81.9 → cognite_neat-0.81.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.81.9"
+version = "0.81.11"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.81.9/cognite/neat/_version.py

@@ -1 +0,0 @@
-__version__ = "0.81.9"

cognite_neat-0.81.9/cognite/neat/graph/extractors/_classic_cdf/_relationships.py

@@ -1,152 +0,0 @@
-import uuid
-from collections.abc import Iterable
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import cast
-from urllib.parse import quote
-
-from cognite.client import CogniteClient
-from cognite.client.data_classes import Relationship, RelationshipList
-from rdflib import RDF, Literal, Namespace
-
-from cognite.neat.constants import DEFAULT_NAMESPACE
-from cognite.neat.graph.extractors._base import BaseExtractor
-from cognite.neat.graph.models import Triple
-
-
-class RelationshipsExtractor(BaseExtractor):
-    def __init__(
-        self,
-        relationships: Iterable[Relationship],
-        namespace: Namespace | None = None,
-    ):
-        self.namespace = namespace or DEFAULT_NAMESPACE
-        self.relationships = relationships
-
-    @classmethod
-    def from_dataset(
-        cls,
-        client: CogniteClient,
-        data_set_external_id: str,
-        namespace: Namespace | None = None,
-    ):
-        return cls(
-            cast(Iterable[Relationship], client.relationships(data_set_external_ids=data_set_external_id)), namespace
-        )
-
-    @classmethod
-    def from_file(cls, file_path: str, namespace: Namespace | None = None):
-        return cls(RelationshipList.load(Path(file_path).read_text()), namespace)
-
-    def extract(self) -> Iterable[Triple]:
-        """Extracts an asset with the given asset_id."""
-        for relationship in self.relationships:
-            yield from self._relationship2triples(relationship, self.namespace)
-
-    @classmethod
-    def _relationship2triples(cls, relationship: Relationship, namespace: Namespace) -> list[Triple]:
-        """Converts an asset to triples."""
-
-        # relationships do not have an internal id, so we generate one
-        id_ = namespace[f"Relationship_{uuid.uuid4()}"]
-
-        # Set rdf type
-        triples: list[Triple] = [(id_, RDF.type, namespace["Relationship"])]
-
-        # Set source and target types
-        if source_type := relationship.source_type:
-            triples.append(
-                (
-                    id_,
-                    namespace.source_type,
-                    namespace[source_type.title()],
-                )
-            )
-
-        if target_type := relationship.target_type:
-            triples.append(
-                (
-                    id_,
-                    namespace.target_type,
-                    namespace[target_type.title()],
-                )
-            )
-
-        # Create attributes
-        if relationship.external_id:
-            triples.append((id_, namespace.external_id, Literal(relationship.external_id)))
-
-        if relationship.source_external_id:
-            triples.append(
-                (
-                    id_,
-                    namespace.source_external_id,
-                    Literal(relationship.source_external_id),
-                )
-            )
-
-        if relationship.target_external_id:
-            triples.append(
-                (
-                    id_,
-                    namespace.target_external_id,
-                    Literal(relationship.target_external_id),
-                )
-            )
-
-        if relationship.start_time:
-            triples.append(
-                (
-                    id_,
-                    namespace.start_time,
-                    Literal(datetime.fromtimestamp(relationship.start_time / 1000, timezone.utc)),
-                )
-            )
-
-        if relationship.end_time:
-            triples.append(
-                (
-                    id_,
-                    namespace.end_time,
-                    Literal(datetime.fromtimestamp(relationship.end_time / 1000, timezone.utc)),
-                )
-            )
-
-        if relationship.created_time:
-            triples.append(
-                (
-                    id_,
-                    namespace.created_time,
-                    Literal(datetime.fromtimestamp(relationship.created_time / 1000, timezone.utc)),
-                )
-            )
-
-        if relationship.last_updated_time:
-            triples.append(
-                (
-                    id_,
-                    namespace.last_updated_time,
-                    Literal(datetime.fromtimestamp(relationship.last_updated_time / 1000, timezone.utc)),
-                )
-            )
-
-        if relationship.confidence:
-            triples.append(
-                (
-                    id_,
-                    namespace.confidence,
-                    Literal(relationship.confidence),
-                )
-            )
-
-        if relationship.labels:
-            for label in relationship.labels:
-                # external_id can create ill-formed URIs, so we create websafe URIs
-                # since labels do not have internal ids, we use the external_id as the id
-                triples.append((id_, namespace.label, namespace[f"Label_{quote(label.dump()['externalId'])}"]))
-
-        # Create connection
-        if relationship.data_set_id:
-            triples.append((id_, namespace.dataset, namespace[f"Dataset_{relationship.data_set_id}"]))
-
-        return triples

cognite_neat-0.81.9/cognite/neat/graph/transformers/__init__.py

@@ -1,5 +0,0 @@
-from ._classic_cdf import AddAssetDepth, AssetFileConnector, AssetSequenceConnector, AssetTimeSeriesConnector
-
-__all__ = ["AddAssetDepth", "AssetTimeSeriesConnector", "AssetSequenceConnector", "AssetFileConnector"]
-
-Transformers = AddAssetDepth | AssetTimeSeriesConnector | AssetSequenceConnector | AssetFileConnector