cognite-neat 0.81.10__tar.gz → 0.81.12__tar.gz

{cognite_neat-0.81.10 → cognite_neat-0.81.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.81.10
+Version: 0.81.12
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.81.12/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.81.12"

{cognite_neat-0.81.10 → cognite_neat-0.81.12}/cognite/neat/graph/extractors/_classic_cdf/_assets.py

@@ -2,7 +2,6 @@ from collections.abc import Iterable
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import cast
-from urllib.parse import quote
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes import Asset, AssetList
@@ -11,7 +10,7 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.utils import create_sha256_hash, string_to_ideal_type
 
 
 class AssetsExtractor(BaseExtractor):
@@ -86,7 +85,9 @@ class AssetsExtractor(BaseExtractor):
             for label in asset.labels:
                 # external_id can create ill-formed URIs, so we create websafe URIs
                 # since labels do not have internal ids, we use the external_id as the id
-                triples.append((id_, namespace.label, namespace[f"Label_{quote(label.dump()['externalId'])}"]))
+                triples.append(
+                    (id_, namespace.label, namespace[f"Label_{create_sha256_hash(label.dump()['externalId'])}"])
+                )
 
         if asset.metadata:
             for key, value in asset.metadata.items():

{cognite_neat-0.81.10 → cognite_neat-0.81.12}/cognite/neat/graph/extractors/_classic_cdf/_labels.py

@@ -2,7 +2,6 @@ from collections.abc import Iterable
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import cast
-from urllib.parse import quote
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes import LabelDefinition, LabelDefinitionList
@@ -11,6 +10,7 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import create_sha256_hash
 
 
 class LabelsExtractor(BaseExtractor):
@@ -44,28 +44,32 @@ class LabelsExtractor(BaseExtractor):
 
     @classmethod
     def _labels2triples(cls, label: LabelDefinition, namespace: Namespace) -> list[Triple]:
-        id_ = namespace[f"Label_{quote(label.dump()['externalId'])}"]
-
-        # Set rdf type
-        triples: list[Triple] = [(id_, RDF.type, namespace.Label)]
+        if label.external_id:
+            id_ = namespace[f"Label_{create_sha256_hash(label.external_id)}"]
 
-        # Create attributes
+            # Set rdf type
+            triples: list[Triple] = [(id_, RDF.type, namespace.Label)]
 
-        if label.external_id:
+            # Create attributes
             triples.append((id_, namespace.external_id, Literal(label.external_id)))
 
-        if label.name:
-            triples.append((id_, namespace.name, Literal(label.name)))
+            if label.name:
+                triples.append((id_, namespace.name, Literal(label.name)))
 
-        if label.description:
-            triples.append((id_, namespace.description, Literal(label.description)))
+            if label.description:
+                triples.append((id_, namespace.description, Literal(label.description)))
 
-        if label.created_time:
-            triples.append(
-                (id_, namespace.created_time, Literal(datetime.fromtimestamp(label.created_time / 1000, timezone.utc)))
-            )
+            if label.created_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.created_time,
+                        Literal(datetime.fromtimestamp(label.created_time / 1000, timezone.utc)),
+                    )
+                )
 
-        if label.data_set_id:
-            triples.append((id_, namespace.data_set_id, namespace[f"Dataset_{label.data_set_id}"]))
+            if label.data_set_id:
+                triples.append((id_, namespace.data_set_id, namespace[f"Dataset_{label.data_set_id}"]))
 
-        return triples
+            return triples
+        return []

cognite_neat-0.81.12/cognite/neat/graph/extractors/_classic_cdf/_relationships.py

@@ -0,0 +1,152 @@
+from collections.abc import Iterable
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import cast
+from urllib.parse import quote
+
+from cognite.client import CogniteClient
+from cognite.client.data_classes import Relationship, RelationshipList
+from rdflib import RDF, Literal, Namespace
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import create_sha256_hash
+
+
+class RelationshipsExtractor(BaseExtractor):
+    def __init__(
+        self,
+        relationships: Iterable[Relationship],
+        namespace: Namespace | None = None,
+    ):
+        self.namespace = namespace or DEFAULT_NAMESPACE
+        self.relationships = relationships
+
+    @classmethod
+    def from_dataset(
+        cls,
+        client: CogniteClient,
+        data_set_external_id: str,
+        namespace: Namespace | None = None,
+    ):
+        return cls(
+            cast(Iterable[Relationship], client.relationships(data_set_external_ids=data_set_external_id)), namespace
+        )
+
+    @classmethod
+    def from_file(cls, file_path: str, namespace: Namespace | None = None):
+        return cls(RelationshipList.load(Path(file_path).read_text()), namespace)
+
+    def extract(self) -> Iterable[Triple]:
+        """Extracts an asset with the given asset_id."""
+        for relationship in self.relationships:
+            yield from self._relationship2triples(relationship, self.namespace)
+
+    @classmethod
+    def _relationship2triples(cls, relationship: Relationship, namespace: Namespace) -> list[Triple]:
+        """Converts an asset to triples."""
+
+        if relationship.external_id and relationship.source_external_id and relationship.target_external_id:
+            # relationships do not have an internal id, so we generate one
+            id_ = namespace[f"Relationship_{create_sha256_hash(relationship.external_id)}"]
+
+            # Set rdf type
+            triples: list[Triple] = [(id_, RDF.type, namespace["Relationship"])]
+
+            # Set source and target types
+            if source_type := relationship.source_type:
+                triples.append(
+                    (
+                        id_,
+                        namespace.source_type,
+                        namespace[source_type.title()],
+                    )
+                )
+
+            if target_type := relationship.target_type:
+                triples.append(
+                    (
+                        id_,
+                        namespace.target_type,
+                        namespace[target_type.title()],
+                    )
+                )
+
+            # Create attributes
+
+            triples.append((id_, namespace.external_id, Literal(relationship.external_id)))
+
+            triples.append(
+                (
+                    id_,
+                    namespace.source_external_id,
+                    Literal(relationship.source_external_id),
+                )
+            )
+
+            triples.append(
+                (
+                    id_,
+                    namespace.target_external_id,
+                    Literal(relationship.target_external_id),
+                )
+            )
+
+            if relationship.start_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.start_time,
+                        Literal(datetime.fromtimestamp(relationship.start_time / 1000, timezone.utc)),
+                    )
+                )
+
+            if relationship.end_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.end_time,
+                        Literal(datetime.fromtimestamp(relationship.end_time / 1000, timezone.utc)),
+                    )
+                )
+
+            if relationship.created_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.created_time,
+                        Literal(datetime.fromtimestamp(relationship.created_time / 1000, timezone.utc)),
+                    )
+                )
+
+            if relationship.last_updated_time:
+                triples.append(
+                    (
+                        id_,
+                        namespace.last_updated_time,
+                        Literal(datetime.fromtimestamp(relationship.last_updated_time / 1000, timezone.utc)),
+                    )
+                )
+
+            if relationship.confidence:
+                triples.append(
+                    (
+                        id_,
+                        namespace.confidence,
+                        Literal(relationship.confidence),
+                    )
+                )
+
+            if relationship.labels:
+                for label in relationship.labels:
+                    # external_id can create ill-formed URIs, so we create websafe URIs
+                    # since labels do not have internal ids, we use the external_id as the id
+                    triples.append((id_, namespace.label, namespace[f"Label_{quote(label.dump()['externalId'])}"]))
+
+            # Create connection
+            if relationship.data_set_id:
+                triples.append((id_, namespace.dataset, namespace[f"Dataset_{relationship.data_set_id}"]))
+
+            return triples
+        return []

{cognite_neat-0.81.10 → cognite_neat-0.81.12}/cognite/neat/graph/stores/_base.py

@@ -14,6 +14,7 @@ from cognite.neat.graph._shared import MIMETypes
 from cognite.neat.graph.extractors import RdfFileExtractor, TripleExtractors
 from cognite.neat.graph.models import Triple
 from cognite.neat.graph.transformers import Transformers
+from cognite.neat.rules.models.entities import ClassEntity
 from cognite.neat.rules.models.information import InformationRules
 from cognite.neat.utils import remove_namespace
 from cognite.neat.utils.auxiliary import local_import
@@ -42,6 +43,8 @@ class NeatGraphStore:
         graph: Graph,
         rules: InformationRules | None = None,
     ):
+        self.rules: InformationRules | None = None
+
         _start = datetime.now(timezone.utc)
         self.graph = graph
         self.provenance = Provenance(
@@ -54,11 +57,30 @@ class NeatGraphStore:
                 )
             ]
         )
-        self.rules = rules
 
-        if self.rules and self.rules.prefixes:
-            self._upsert_prefixes(self.rules.prefixes)
+        if rules:
+            self.rules = rules
             self.base_namespace = self.rules.metadata.namespace
+            self.provenance.append(
+                Change.record(
+                    activity=f"{type(self)}.rules",
+                    start=_start,
+                    end=datetime.now(timezone.utc),
+                    description=f"Added rules to graph store as {type(self.rules).__name__}",
+                )
+            )
+
+            if self.rules.prefixes:
+                self._upsert_prefixes(self.rules.prefixes)
+                self.provenance.append(
+                    Change.record(
+                        activity=f"{type(self).__name__}._upsert_prefixes",
+                        start=_start,
+                        end=datetime.now(timezone.utc),
+                        description="Upsert prefixes to graph store",
+                    )
+                )
+
         else:
             self.base_namespace = DEFAULT_NAMESPACE
 
@@ -144,6 +166,23 @@ class NeatGraphStore:
             )
         )
 
+    def read(self, class_: str) -> list[tuple[str, str, str]]:
+        """Read instances for given view from the graph store."""
+        # PLACEHOLDER: Implement reading instances for a given view
+        # not yet developed
+
+        if not self.rules:
+            warnings.warn("No rules found for the graph store, returning empty list.", stacklevel=2)
+            return []
+
+        class_entity = ClassEntity(prefix=self.rules.metadata.prefix, suffix=class_)
+
+        if class_entity not in [definition.class_ for definition in self.rules.classes.data]:
+            warnings.warn("Desired type not found in graph!", stacklevel=2)
+            return []
+
+        return []
+
     def _parse_file(
         self,
         filepath: Path,

{cognite_neat-0.81.10 → cognite_neat-0.81.12}/cognite/neat/graph/transformers/__init__.py

@@ -2,6 +2,7 @@ from ._classic_cdf import (
     AddAssetDepth,
     AssetEventConnector,
     AssetFileConnector,
+    AssetRelationshipConnector,
     AssetSequenceConnector,
     AssetTimeSeriesConnector,
 )
@@ -12,8 +13,14 @@ __all__ = [
     "AssetSequenceConnector",
     "AssetFileConnector",
     "AssetEventConnector",
+    "AssetRelationshipConnector",
 ]
 
 Transformers = (
-    AddAssetDepth | AssetTimeSeriesConnector | AssetSequenceConnector | AssetFileConnector | AssetEventConnector
+    AddAssetDepth
+    | AssetTimeSeriesConnector
+    | AssetSequenceConnector
+    | AssetFileConnector
+    | AssetEventConnector
+    | AssetRelationshipConnector
 )

{cognite_neat-0.81.10 → cognite_neat-0.81.12}/cognite/neat/graph/transformers/_classic_cdf.py

@@ -205,3 +205,64 @@ class AssetEventConnector(BaseTransformer):
                 # files can be connected to multiple assets in the graph
                 for (asset_id,) in cast(list[tuple], assets_id_res):
                     graph.add((asset_id, DEFAULT_NAMESPACE.event, event_id))
+
+
+class AssetRelationshipConnector(BaseTransformer):
+    description: str = "Connects assets via relationships"
+    _use_only_once: bool = True
+    _need_changes = frozenset(
+        {str(extractors.AssetsExtractor.__name__), str(extractors.RelationshipsExtractor.__name__)}
+    )
+    _asset_template: str = """SELECT ?source ?target WHERE {{
+                              <{relationship_id}> <{relationship_source_xid_prop}> ?source_xid .
+                              ?source <{asset_xid_property}> ?source_xid .
+                              ?source a <{asset_type}> .
+
+                              <{relationship_id}> <{relationship_target_xid_prop}> ?target_xid .
+                              ?target <{asset_xid_property}> ?target_xid .
+                              ?target a <{asset_type}> .}}"""
+
+    def __init__(
+        self,
+        asset_type: URIRef | None = None,
+        relationship_type: URIRef | None = None,
+        relationship_source_xid_prop: URIRef | None = None,
+        relationship_target_xid_prop: URIRef | None = None,
+        asset_xid_property: URIRef | None = None,
+    ):
+        self.asset_type = asset_type or DEFAULT_NAMESPACE.Asset
+        self.relationship_type = relationship_type or DEFAULT_NAMESPACE.Relationship
+        self.relationship_source_xid_prop = relationship_source_xid_prop or DEFAULT_NAMESPACE.source_external_id
+        self.relationship_target_xid_prop = relationship_target_xid_prop or DEFAULT_NAMESPACE.target_external_id
+        self.asset_xid_property = asset_xid_property or DEFAULT_NAMESPACE.external_id
+
+    def transform(self, graph: Graph) -> None:
+        for relationship_id_result in graph.query(
+            f"SELECT DISTINCT ?relationship_id WHERE {{?relationship_id a <{self.relationship_type}>}}"
+        ):
+            relationship_id: URIRef = cast(tuple, relationship_id_result)[0]
+
+            if assets_id_res := list(
+                graph.query(
+                    self._asset_template.format(
+                        relationship_id=relationship_id,
+                        asset_xid_property=self.asset_xid_property,
+                        relationship_source_xid_prop=self.relationship_source_xid_prop,
+                        relationship_target_xid_prop=self.relationship_target_xid_prop,
+                        asset_type=self.asset_type,
+                    )
+                )
+            ):
+                # files can be connected to multiple assets in the graph
+                for source_asset_id, target_asset_id in cast(list[tuple], assets_id_res):
+                    # create a relationship between the two assets
+                    graph.add((source_asset_id, DEFAULT_NAMESPACE.relationship, relationship_id))
+                    graph.add((target_asset_id, DEFAULT_NAMESPACE.relationship, relationship_id))
+
+                    # add source and target to the relationship
+                    graph.add((relationship_id, DEFAULT_NAMESPACE.source, source_asset_id))
+                    graph.add((relationship_id, DEFAULT_NAMESPACE.target, target_asset_id))
+
+                    # remove properties that are not needed, specifically the external ids
+                    graph.remove((relationship_id, self.relationship_source_xid_prop, None))
+                    graph.remove((relationship_id, self.relationship_target_xid_prop, None))

{cognite_neat-0.81.10 → cognite_neat-0.81.12}/cognite/neat/rules/analysis/_information_rules.py

@@ -8,7 +8,7 @@ import pandas as pd
 from pydantic import ValidationError
 
 from cognite.neat.rules.models import SchemaCompleteness
-from cognite.neat.rules.models._rdfpath import TransformationRuleType
+from cognite.neat.rules.models._rdfpath import RDFPath
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, ParentClassEntity, ReferenceEntity
 from cognite.neat.rules.models.information import InformationClass, InformationProperty, InformationRules
 from cognite.neat.utils.utils import get_inheritance_path
@@ -191,7 +191,7 @@ class InformationArchitectRulesAnalysis(BaseAnalysis):
                     )
                     continue
 
-                if (only_rdfpath and property_.rule_type == TransformationRuleType.rdfpath) or not only_rdfpath:
+                if (only_rdfpath and isinstance(property_.transformation, RDFPath)) or not only_rdfpath:
                     processed_properties[property_.property_] = property_
             class_property_pairs[class_] = processed_properties
 

{cognite_neat-0.81.10 → cognite_neat-0.81.12}/cognite/neat/rules/models/information/_rules.py

@@ -23,13 +23,8 @@ from cognite.neat.rules.models._base import (
     SheetList,
 )
 from cognite.neat.rules.models._rdfpath import (
-    AllReferences,
-    Hop,
-    RawLookup,
-    SingleProperty,
-    SPARQLQuery,
+    RDFPath,
     TransformationRuleType,
-    Traversal,
     parse_rule,
 )
 from cognite.neat.rules.models._types import (
@@ -156,9 +151,7 @@ class InformationProperty(SheetEntity):
         default: Default value of the property
         reference: Reference to the source of the information, HTTP URI
         match_type: The match type of the resource being described and the source entity.
-        rule_type: Rule type for the transformation from source to target representation
-                   of knowledge graph. Defaults to None (no transformation)
-        rule: Actual rule for the transformation from source to target representation of
+        transformation: Actual rule for the transformation from source to target representation of
               knowledge graph. Defaults to None (no transformation)
     """
 
@@ -174,10 +167,7 @@ class InformationProperty(SheetEntity):
     default: Any | None = Field(alias="Default", default=None)
     reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
     match_type: MatchType | None = Field(alias="Match Type", default=None)
-    rule_type: str | TransformationRuleType | None = Field(alias="Rule Type", default=None)
-    rule: str | AllReferences | SingleProperty | Hop | RawLookup | SPARQLQuery | Traversal | None = Field(
-        alias="Rule", default=None
-    )
+    transformation: str | RDFPath | None = Field(alias="Transformation", default=None)
     comment: str | None = Field(alias="Comment", default=None)
 
     @field_serializer("max_count", when_used="json-unless-none")
@@ -193,15 +183,10 @@ class InformationProperty(SheetEntity):
         return value
 
     @model_validator(mode="after")
-    def is_valid_rule(self):
-        # TODO: Can we skip rule_type and simply try to parse the rule and if it fails, raise an error?
-        if self.rule_type:
-            self.rule_type = self.rule_type.lower()
-            if not self.rule:
-                raise exceptions.RuleTypeProvidedButRuleMissing(
-                    self.property_, self.class_, self.rule_type
-                ).to_pydantic_custom_error()
-            self.rule = parse_rule(self.rule, self.rule_type)
+    def generate_valid_transformation(self):
+        # TODO: Currently only supporting RDFpath
+        if self.transformation:
+            self.transformation = parse_rule(self.transformation, TransformationRuleType.rdfpath)
         return self
 
     @model_validator(mode="after")

{cognite_neat-0.81.10 → cognite_neat-0.81.12}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.81.10"
+version = "0.81.12"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.81.10/cognite/neat/_version.py

@@ -1 +0,0 @@
-__version__ = "0.81.10"

cognite_neat-0.81.10/cognite/neat/graph/extractors/_classic_cdf/_relationships.py

@@ -1,152 +0,0 @@
-import uuid
-from collections.abc import Iterable
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import cast
-from urllib.parse import quote
-
-from cognite.client import CogniteClient
-from cognite.client.data_classes import Relationship, RelationshipList
-from rdflib import RDF, Literal, Namespace
-
-from cognite.neat.constants import DEFAULT_NAMESPACE
-from cognite.neat.graph.extractors._base import BaseExtractor
-from cognite.neat.graph.models import Triple
-
-
-class RelationshipsExtractor(BaseExtractor):
-    def __init__(
-        self,
-        relationships: Iterable[Relationship],
-        namespace: Namespace | None = None,
-    ):
-        self.namespace = namespace or DEFAULT_NAMESPACE
-        self.relationships = relationships
-
-    @classmethod
-    def from_dataset(
-        cls,
-        client: CogniteClient,
-        data_set_external_id: str,
-        namespace: Namespace | None = None,
-    ):
-        return cls(
-            cast(Iterable[Relationship], client.relationships(data_set_external_ids=data_set_external_id)), namespace
-        )
-
-    @classmethod
-    def from_file(cls, file_path: str, namespace: Namespace | None = None):
-        return cls(RelationshipList.load(Path(file_path).read_text()), namespace)
-
-    def extract(self) -> Iterable[Triple]:
-        """Extracts an asset with the given asset_id."""
-        for relationship in self.relationships:
-            yield from self._relationship2triples(relationship, self.namespace)
-
-    @classmethod
-    def _relationship2triples(cls, relationship: Relationship, namespace: Namespace) -> list[Triple]:
-        """Converts an asset to triples."""
-
-        # relationships do not have an internal id, so we generate one
-        id_ = namespace[f"Relationship_{uuid.uuid4()}"]
-
-        # Set rdf type
-        triples: list[Triple] = [(id_, RDF.type, namespace["Relationship"])]
-
-        # Set source and target types
-        if source_type := relationship.source_type:
-            triples.append(
-                (
-                    id_,
-                    namespace.source_type,
-                    namespace[source_type.title()],
-                )
-            )
-
-        if target_type := relationship.target_type:
-            triples.append(
-                (
-                    id_,
-                    namespace.target_type,
-                    namespace[target_type.title()],
-                )
-            )
-
-        # Create attributes
-        if relationship.external_id:
-            triples.append((id_, namespace.external_id, Literal(relationship.external_id)))
-
-        if relationship.source_external_id:
-            triples.append(
-                (
-                    id_,
-                    namespace.source_external_id,
-                    Literal(relationship.source_external_id),
-                )
-            )
-
-        if relationship.target_external_id:
-            triples.append(
-                (
-                    id_,
-                    namespace.target_external_id,
-                    Literal(relationship.target_external_id),
-                )
-            )
-
-        if relationship.start_time:
-            triples.append(
-                (
-                    id_,
-                    namespace.start_time,
-                    Literal(datetime.fromtimestamp(relationship.start_time / 1000, timezone.utc)),
-                )
-            )
-
-        if relationship.end_time:
-            triples.append(
-                (
-                    id_,
-                    namespace.end_time,
-                    Literal(datetime.fromtimestamp(relationship.end_time / 1000, timezone.utc)),
-                )
-            )
-
-        if relationship.created_time:
-            triples.append(
-                (
-                    id_,
-                    namespace.created_time,
-                    Literal(datetime.fromtimestamp(relationship.created_time / 1000, timezone.utc)),
-                )
-            )
-
-        if relationship.last_updated_time:
-            triples.append(
-                (
-                    id_,
-                    namespace.last_updated_time,
-                    Literal(datetime.fromtimestamp(relationship.last_updated_time / 1000, timezone.utc)),
-                )
-            )
-
-        if relationship.confidence:
-            triples.append(
-                (
-                    id_,
-                    namespace.confidence,
-                    Literal(relationship.confidence),
-                )
-            )
-
-        if relationship.labels:
-            for label in relationship.labels:
-                # external_id can create ill-formed URIs, so we create websafe URIs
-                # since labels do not have internal ids, we use the external_id as the id
-                triples.append((id_, namespace.label, namespace[f"Label_{quote(label.dump()['externalId'])}"]))
-
-        # Create connection
-        if relationship.data_set_id:
-            triples.append((id_, namespace.dataset, namespace[f"Dataset_{relationship.data_set_id}"]))
-
-        return triples