cognite-neat 0.81.10__py3-none-any.whl → 0.81.11__py3-none-any.whl

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.81.10"
+__version__ = "0.81.11"

cognite/neat/graph/extractors/_classic_cdf/_assets.py

@@ -2,7 +2,6 @@ from collections.abc import Iterable
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import cast
-from urllib.parse import quote
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes import Asset, AssetList
@@ -11,7 +10,7 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.utils import create_sha256_hash, string_to_ideal_type
 
 
 class AssetsExtractor(BaseExtractor):
@@ -86,7 +85,9 @@ class AssetsExtractor(BaseExtractor):
             for label in asset.labels:
                 # external_id can create ill-formed URIs, so we create websafe URIs
                 # since labels do not have internal ids, we use the external_id as the id
-                triples.append((id_, namespace.label, namespace[f"Label_{quote(label.dump()['externalId'])}"]))
+                triples.append(
+                    (id_, namespace.label, namespace[f"Label_{create_sha256_hash(label.dump()['externalId'])}"])
+                )
 
         if asset.metadata:
             for key, value in asset.metadata.items():

cognite/neat/graph/extractors/_classic_cdf/_labels.py

@@ -2,7 +2,6 @@ from collections.abc import Iterable
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import cast
-from urllib.parse import quote
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes import LabelDefinition, LabelDefinitionList
@@ -11,6 +10,7 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import create_sha256_hash
 
 
 class LabelsExtractor(BaseExtractor):
@@ -44,28 +44,32 @@ class LabelsExtractor(BaseExtractor):
 
     @classmethod
     def _labels2triples(cls, label: LabelDefinition, namespace: Namespace) -> list[Triple]:
-        id_ = namespace[f"Label_{quote(label.dump()['externalId'])}"]
-
-        # Set rdf type
-        triples: list[Triple] = [(id_, RDF.type, namespace.Label)]
+        if label.external_id:
+            id_ = namespace[f"Label_{create_sha256_hash(label.external_id)}"]
 
-        # Create attributes
+            # Set rdf type
+            triples: list[Triple] = [(id_, RDF.type, namespace.Label)]
 
-        if label.external_id:
+            # Create attributes
             triples.append((id_, namespace.external_id, Literal(label.external_id)))
 
-        if label.name:
-            triples.append((id_, namespace.name, Literal(label.name)))
+            if label.name:
+                triples.append((id_, namespace.name, Literal(label.name)))
 
-        if label.description:
-            triples.append((id_, namespace.description, Literal(label.description)))
+            if label.description:
+                triples.append((id_, namespace.description, Literal(label.description)))
 
-        if label.created_time:
-            triples.append(
-                (id_, namespace.created_time, Literal(datetime.fromtimestamp(label.created_time / 1000, timezone.utc)))
-            )
+            if label.created_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.created_time,
+                        Literal(datetime.fromtimestamp(label.created_time / 1000, timezone.utc)),
+                    )
+                )
 
-        if label.data_set_id:
-            triples.append((id_, namespace.data_set_id, namespace[f"Dataset_{label.data_set_id}"]))
+            if label.data_set_id:
+                triples.append((id_, namespace.data_set_id, namespace[f"Dataset_{label.data_set_id}"]))
 
-        return triples
+            return triples
+        return []

cognite/neat/graph/extractors/_classic_cdf/_relationships.py

@@ -1,4 +1,3 @@
-import uuid
 from collections.abc import Iterable
 from datetime import datetime, timezone
 from pathlib import Path
@@ -12,6 +11,7 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import create_sha256_hash
 
 
 class RelationshipsExtractor(BaseExtractor):
@@ -47,36 +47,36 @@ class RelationshipsExtractor(BaseExtractor):
     def _relationship2triples(cls, relationship: Relationship, namespace: Namespace) -> list[Triple]:
         """Converts an asset to triples."""
 
-        # relationships do not have an internal id, so we generate one
-        id_ = namespace[f"Relationship_{uuid.uuid4()}"]
-
-        # Set rdf type
-        triples: list[Triple] = [(id_, RDF.type, namespace["Relationship"])]
-
-        # Set source and target types
-        if source_type := relationship.source_type:
-            triples.append(
-                (
-                    id_,
-                    namespace.source_type,
-                    namespace[source_type.title()],
+        if relationship.external_id and relationship.source_external_id and relationship.target_external_id:
+            # relationships do not have an internal id, so we generate one
+            id_ = namespace[f"Relationship_{create_sha256_hash(relationship.external_id)}"]
+
+            # Set rdf type
+            triples: list[Triple] = [(id_, RDF.type, namespace["Relationship"])]
+
+            # Set source and target types
+            if source_type := relationship.source_type:
+                triples.append(
+                    (
+                        id_,
+                        namespace.source_type,
+                        namespace[source_type.title()],
+                    )
                 )
-            )
 
-        if target_type := relationship.target_type:
-            triples.append(
-                (
-                    id_,
-                    namespace.target_type,
-                    namespace[target_type.title()],
+            if target_type := relationship.target_type:
+                triples.append(
+                    (
+                        id_,
+                        namespace.target_type,
+                        namespace[target_type.title()],
+                    )
                 )
-            )
 
-        # Create attributes
-        if relationship.external_id:
+            # Create attributes
+
             triples.append((id_, namespace.external_id, Literal(relationship.external_id)))
 
-        if relationship.source_external_id:
             triples.append(
                 (
                     id_,
@@ -85,7 +85,6 @@ class RelationshipsExtractor(BaseExtractor):
                 )
             )
 
-        if relationship.target_external_id:
             triples.append(
                 (
                     id_,
@@ -94,59 +93,60 @@ class RelationshipsExtractor(BaseExtractor):
                 )
             )
 
-        if relationship.start_time:
-            triples.append(
-                (
-                    id_,
-                    namespace.start_time,
-                    Literal(datetime.fromtimestamp(relationship.start_time / 1000, timezone.utc)),
+            if relationship.start_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.start_time,
+                        Literal(datetime.fromtimestamp(relationship.start_time / 1000, timezone.utc)),
+                    )
                 )
-            )
 
-        if relationship.end_time:
-            triples.append(
-                (
-                    id_,
-                    namespace.end_time,
-                    Literal(datetime.fromtimestamp(relationship.end_time / 1000, timezone.utc)),
+            if relationship.end_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.end_time,
+                        Literal(datetime.fromtimestamp(relationship.end_time / 1000, timezone.utc)),
+                    )
                 )
-            )
 
-        if relationship.created_time:
-            triples.append(
-                (
-                    id_,
-                    namespace.created_time,
-                    Literal(datetime.fromtimestamp(relationship.created_time / 1000, timezone.utc)),
+            if relationship.created_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.created_time,
+                        Literal(datetime.fromtimestamp(relationship.created_time / 1000, timezone.utc)),
+                    )
                 )
-            )
 
-        if relationship.last_updated_time:
-            triples.append(
-                (
-                    id_,
-                    namespace.last_updated_time,
-                    Literal(datetime.fromtimestamp(relationship.last_updated_time / 1000, timezone.utc)),
+            if relationship.last_updated_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.last_updated_time,
+                        Literal(datetime.fromtimestamp(relationship.last_updated_time / 1000, timezone.utc)),
+                    )
                 )
-            )
 
-        if relationship.confidence:
-            triples.append(
-                (
-                    id_,
-                    namespace.confidence,
-                    Literal(relationship.confidence),
+            if relationship.confidence:
+                triples.append(
+                    (
+                        id_,
+                        namespace.confidence,
+                        Literal(relationship.confidence),
+                    )
                 )
-            )
 
-        if relationship.labels:
-            for label in relationship.labels:
-                # external_id can create ill-formed URIs, so we create websafe URIs
-                # since labels do not have internal ids, we use the external_id as the id
-                triples.append((id_, namespace.label, namespace[f"Label_{quote(label.dump()['externalId'])}"]))
+            if relationship.labels:
+                for label in relationship.labels:
+                    # external_id can create ill-formed URIs, so we create websafe URIs
+                    # since labels do not have internal ids, we use the external_id as the id
+                    triples.append((id_, namespace.label, namespace[f"Label_{quote(label.dump()['externalId'])}"]))
 
-        # Create connection
-        if relationship.data_set_id:
-            triples.append((id_, namespace.dataset, namespace[f"Dataset_{relationship.data_set_id}"]))
+            # Create connection
+            if relationship.data_set_id:
+                triples.append((id_, namespace.dataset, namespace[f"Dataset_{relationship.data_set_id}"]))
 
-        return triples
+            return triples
+        return []

cognite/neat/graph/transformers/__init__.py

@@ -2,6 +2,7 @@ from ._classic_cdf import (
     AddAssetDepth,
     AssetEventConnector,
     AssetFileConnector,
+    AssetRelationshipConnector,
     AssetSequenceConnector,
     AssetTimeSeriesConnector,
 )
@@ -12,8 +13,14 @@ __all__ = [
     "AssetSequenceConnector",
     "AssetFileConnector",
     "AssetEventConnector",
+    "AssetRelationshipConnector",
 ]
 
 Transformers = (
-    AddAssetDepth | AssetTimeSeriesConnector | AssetSequenceConnector | AssetFileConnector | AssetEventConnector
+    AddAssetDepth
+    | AssetTimeSeriesConnector
+    | AssetSequenceConnector
+    | AssetFileConnector
+    | AssetEventConnector
+    | AssetRelationshipConnector
 )

cognite/neat/graph/transformers/_classic_cdf.py

@@ -205,3 +205,64 @@ class AssetEventConnector(BaseTransformer):
                 # files can be connected to multiple assets in the graph
                 for (asset_id,) in cast(list[tuple], assets_id_res):
                     graph.add((asset_id, DEFAULT_NAMESPACE.event, event_id))
+
+
+class AssetRelationshipConnector(BaseTransformer):
+    description: str = "Connects assets via relationships"
+    _use_only_once: bool = True
+    _need_changes = frozenset(
+        {str(extractors.AssetsExtractor.__name__), str(extractors.RelationshipsExtractor.__name__)}
+    )
+    _asset_template: str = """SELECT ?source ?target WHERE {{
+                              <{relationship_id}> <{relationship_source_xid_prop}> ?source_xid .
+                              ?source <{asset_xid_property}> ?source_xid .
+                              ?source a <{asset_type}> .
+
+                              <{relationship_id}> <{relationship_target_xid_prop}> ?target_xid .
+                              ?target <{asset_xid_property}> ?target_xid .
+                              ?target a <{asset_type}> .}}"""
+
+    def __init__(
+        self,
+        asset_type: URIRef | None = None,
+        relationship_type: URIRef | None = None,
+        relationship_source_xid_prop: URIRef | None = None,
+        relationship_target_xid_prop: URIRef | None = None,
+        asset_xid_property: URIRef | None = None,
+    ):
+        self.asset_type = asset_type or DEFAULT_NAMESPACE.Asset
+        self.relationship_type = relationship_type or DEFAULT_NAMESPACE.Relationship
+        self.relationship_source_xid_prop = relationship_source_xid_prop or DEFAULT_NAMESPACE.source_external_id
+        self.relationship_target_xid_prop = relationship_target_xid_prop or DEFAULT_NAMESPACE.target_external_id
+        self.asset_xid_property = asset_xid_property or DEFAULT_NAMESPACE.external_id
+
+    def transform(self, graph: Graph) -> None:
+        for relationship_id_result in graph.query(
+            f"SELECT DISTINCT ?relationship_id WHERE {{?relationship_id a <{self.relationship_type}>}}"
+        ):
+            relationship_id: URIRef = cast(tuple, relationship_id_result)[0]
+
+            if assets_id_res := list(
+                graph.query(
+                    self._asset_template.format(
+                        relationship_id=relationship_id,
+                        asset_xid_property=self.asset_xid_property,
+                        relationship_source_xid_prop=self.relationship_source_xid_prop,
+                        relationship_target_xid_prop=self.relationship_target_xid_prop,
+                        asset_type=self.asset_type,
+                    )
+                )
+            ):
+                # files can be connected to multiple assets in the graph
+                for source_asset_id, target_asset_id in cast(list[tuple], assets_id_res):
+                    # create a relationship between the two assets
+                    graph.add((source_asset_id, DEFAULT_NAMESPACE.relationship, relationship_id))
+                    graph.add((target_asset_id, DEFAULT_NAMESPACE.relationship, relationship_id))
+
+                    # add source and target to the relationship
+                    graph.add((relationship_id, DEFAULT_NAMESPACE.source, source_asset_id))
+                    graph.add((relationship_id, DEFAULT_NAMESPACE.target, target_asset_id))
+
+                    # remove properties that are not needed, specifically the external ids
+                    graph.remove((relationship_id, self.relationship_source_xid_prop, None))
+                    graph.remove((relationship_id, self.relationship_target_xid_prop, None))

{cognite_neat-0.81.10.dist-info → cognite_neat-0.81.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.81.10
+Version: 0.81.11
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

{cognite_neat-0.81.10.dist-info → cognite_neat-0.81.11.dist-info}/RECORD

@@ -1,5 +1,5 @@
 cognite/neat/__init__.py,sha256=v-rRiDOgZ3sQSMQKq0vgUQZvpeOkoHFXissAx6Ktg84,61
-cognite/neat/_version.py,sha256=kfoIPOj8SqOTkglqTypeVhT7HKM-I90zJCvSL7WOcg8,24
+cognite/neat/_version.py,sha256=j--riKojvVEPIAPPLvKSvPrl9iDUa3FZVs9awN4iAfo,24
 cognite/neat/app/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/app/api/asgi/metrics.py,sha256=nxFy7L5cChTI0a-zkCiJ59Aq8yLuIJp5c9Dg0wRXtV0,152
 cognite/neat/app/api/configuration.py,sha256=2U5M6M252swvQPQyooA1EBzFUZNtcTmuSaywfJDgckM,4232
@@ -56,11 +56,11 @@ cognite/neat/graph/exceptions.py,sha256=R6pyOH774n9w2x_X_nrUr8OMAdjJMf_XPIqAvxIQ
 cognite/neat/graph/extractors/__init__.py,sha256=ozXL6ZLK36wp3uX4UACRVs6rbvynQg2JQlDgL1UM1Wk,1025
 cognite/neat/graph/extractors/_base.py,sha256=TOXDnlqske8DgnJwA0THDVRgmR79Acjm56yF0E-2w7I,356
 cognite/neat/graph/extractors/_classic_cdf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cognite/neat/graph/extractors/_classic_cdf/_assets.py,sha256=8FQvJqi8nclkQJ7YmSo4yNqs9uExaoCn_whMW8cIAx0,3855
+cognite/neat/graph/extractors/_classic_cdf/_assets.py,sha256=6e_glBuYl6ArBabOblERQ57g55qsiA0UHrmHVvBqk_s,3895
 cognite/neat/graph/extractors/_classic_cdf/_events.py,sha256=Z0vPcyOz4mCwY0Dqa5wAQZjczO1dbTUGM0X4Y10NLGQ,3995
 cognite/neat/graph/extractors/_classic_cdf/_files.py,sha256=-6nCkXUCAnDsv4eDFDEiQ-U4SGhmW1VLxZJFUcszqjU,4831
-cognite/neat/graph/extractors/_classic_cdf/_labels.py,sha256=wm7JFmsk7sHsOVpTsGBE0wargIuHD09Xu-OHK_Bm20g,2386
-cognite/neat/graph/extractors/_classic_cdf/_relationships.py,sha256=n7gISeyhLjiaWYLWWRj20jmaYgdvJBdYSiZ0G8ZW6mk,5035
+cognite/neat/graph/extractors/_classic_cdf/_labels.py,sha256=nGQyrOZrZzxniIKWROzS1jYrLDT861NARyi-VM_u9-0,2587
+cognite/neat/graph/extractors/_classic_cdf/_relationships.py,sha256=w16hu_REIFEVEenhdxEInOmgCouZaBwxXSlxl9_7vTA,5398
 cognite/neat/graph/extractors/_classic_cdf/_sequences.py,sha256=o4yxkf81FGFrKkflvlyDYie05fTYsT_LcRFM63OTVCI,3406
 cognite/neat/graph/extractors/_classic_cdf/_timeseries.py,sha256=KTYmL8vhXijlmkN1UFQrGpaCllpRekr1y55SoLhlLbg,4559
 cognite/neat/graph/extractors/_dexpi.py,sha256=CYSLt0Fl7Y2RCqOfIAT0N8Cjs-Yu2lRLvB13axtAaWw,9384
@@ -76,9 +76,9 @@ cognite/neat/graph/stores/__init__.py,sha256=G-VG_YwfRt1kuPao07PDJyZ3w_0-eguzLUM
 cognite/neat/graph/stores/_base.py,sha256=-MwqfVCBd4j4xWaI7zOvypsGqtbET-2eKQKf36DO0A0,11001
 cognite/neat/graph/stores/_oxrdflib.py,sha256=A5zeRm5_e8ui_ihGpgstRDg_N7qcLZ3QZBRGrOXSGI0,9569
 cognite/neat/graph/stores/_provenance.py,sha256=Hr9WBhFj-eoet4czL8XSBGYnu9Yn66YsTgH_G0n3QpY,3293
-cognite/neat/graph/transformers/__init__.py,sha256=ZlTjOGX1ly4AlYvc4-D-xoTonkP5OUuJF1t8cj-EUac,440
+cognite/neat/graph/transformers/__init__.py,sha256=wXrNSyJNGnis3haaCKVPZ5y5kKSUsOUHnh-860ekatk,555
 cognite/neat/graph/transformers/_base.py,sha256=b37Ek-9njuM5pTR_3XhnxCMrg_ip_2BMwM7ZhKpAAlw,328
-cognite/neat/graph/transformers/_classic_cdf.py,sha256=fsKwbm__5YpgeOwbUbbFatK5_VBWdZo5F8k97zX0nxw,8876
+cognite/neat/graph/transformers/_classic_cdf.py,sha256=ZHHJU-1-lXeufuZJSuDa2Zmu56PS9PcGeFYI91VZNI4,12214
 cognite/neat/issues.py,sha256=pxQfqfBseMDE8JM0iqZnkLXngeyeFfT0TFtu1UuAd4c,4629
 cognite/neat/legacy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/legacy/graph/__init__.py,sha256=31uTeejWOSd-I8iUG8GOZFhHZcQCsBitJ6X8vu2r1nU,73
@@ -297,8 +297,8 @@ cognite/neat/workflows/steps_registry.py,sha256=fkTX14ZA7_gkUYfWIlx7A1XbCidvqR23
 cognite/neat/workflows/tasks.py,sha256=dqlJwKAb0jlkl7abbY8RRz3m7MT4SK8-7cntMWkOYjw,788
 cognite/neat/workflows/triggers.py,sha256=_BLNplzoz0iic367u1mhHMHiUrCwP-SLK6_CZzfODX0,7071
 cognite/neat/workflows/utils.py,sha256=gKdy3RLG7ctRhbCRwaDIWpL9Mi98zm56-d4jfHDqP1E,453
-cognite_neat-0.81.10.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-0.81.10.dist-info/METADATA,sha256=PJxDNxDQFz5LZWpOfU8ud4ihSmj3lNlF4996gUUR8I0,9291
-cognite_neat-0.81.10.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cognite_neat-0.81.10.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
-cognite_neat-0.81.10.dist-info/RECORD,,
+cognite_neat-0.81.11.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-0.81.11.dist-info/METADATA,sha256=MmBADLlk7wOEvD_MHykgRKYfjDJIoVYbf8WeGTeF9ac,9291
+cognite_neat-0.81.11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cognite_neat-0.81.11.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
+cognite_neat-0.81.11.dist-info/RECORD,,