cognite-neat 0.89.0__tar.gz → 0.90.0__tar.gz

{cognite_neat-0.89.0 → cognite_neat-0.90.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.89.0
+Version: 0.90.0
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.90.0/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.90.0"

{cognite_neat-0.89.0 → cognite_neat-0.90.0}/cognite/neat/constants.py

@@ -7,11 +7,11 @@ from cognite import neat
 PACKAGE_DIRECTORY = Path(neat.__file__).parent
 
 
-EXAMPLE_RULES = PACKAGE_DIRECTORY / "legacy" / "rules" / "examples"
-EXAMPLE_GRAPHS = PACKAGE_DIRECTORY / "legacy" / "graph" / "examples"
-_OLD_WORKFLOWS = PACKAGE_DIRECTORY / "legacy" / "workflows" / "examples"
+EXAMPLE_RULES = PACKAGE_DIRECTORY / "rules" / "examples"
+EXAMPLE_GRAPHS = PACKAGE_DIRECTORY / "graph" / "examples"
 EXAMPLE_WORKFLOWS = PACKAGE_DIRECTORY / "workflows" / "examples"
 
+DEFAULT_SPACE_URI = "http://purl.org/cognite/{space}#"
 DEFAULT_NAMESPACE = Namespace("http://purl.org/cognite/neat#")
 
 

{cognite_neat-0.89.0 → cognite_neat-0.90.0}/cognite/neat/graph/extractors/__init__.py

@@ -7,6 +7,7 @@ from ._classic_cdf._relationships import RelationshipsExtractor
 from ._classic_cdf._sequences import SequencesExtractor
 from ._classic_cdf._timeseries import TimeSeriesExtractor
 from ._dexpi import DexpiExtractor
+from ._dms import DMSExtractor
 from ._mock_graph_generator import MockGraphGenerator
 from ._rdf_file import RdfFileExtractor
 
@@ -22,6 +23,7 @@ __all__ = [
     "LabelsExtractor",
     "RdfFileExtractor",
     "DexpiExtractor",
+    "DMSExtractor",
 ]
 
 

{cognite_neat-0.89.0 → cognite_neat-0.90.0}/cognite/neat/graph/extractors/_classic_cdf/_base.py

@@ -5,7 +5,7 @@ from collections.abc import Callable, Iterable, Set
 from typing import Generic, TypeVar
 
 from cognite.client.data_classes._base import CogniteResource
-from rdflib import Literal, Namespace, URIRef
+from rdflib import XSD, Literal, Namespace, URIRef
 
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
@@ -93,7 +93,7 @@ class ClassicCDFExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
                         Literal(string_to_ideal_type(value)),
                     )
         else:
-            yield id_, self.namespace.metadata, Literal(json.dumps(metadata))
+            yield id_, self.namespace.metadata, Literal(json.dumps(metadata), datatype=XSD._NS["json"])
 
     def _get_rdf_type(self, item: T_CogniteResource) -> str:
         type_ = self._default_rdf_type

cognite_neat-0.90.0/cognite/neat/graph/extractors/_dms.py

@@ -0,0 +1,158 @@
+from collections.abc import Iterable, Iterator
+from typing import cast
+
+from cognite.client import CogniteClient
+from cognite.client import data_modeling as dm
+from cognite.client.data_classes.data_modeling import DataModelIdentifier
+from cognite.client.data_classes.data_modeling.instances import Instance, PropertyValue
+from rdflib import RDF, XSD, Literal, Namespace, URIRef
+
+from cognite.neat.constants import DEFAULT_SPACE_URI
+from cognite.neat.graph.models import Triple
+from cognite.neat.issues.errors import ResourceRetrievalError
+
+from ._base import BaseExtractor
+
+
+class DMSExtractor(BaseExtractor):
+    """Extract data from Cognite Data Fusion DMS instances into Neat.
+
+    Args:
+        items: The items to extract.
+        total: The total number of items to extract. If provided, this will be used to estimate the progress.
+        limit: The maximum number of items to extract.
+        overwrite_namespace: If provided, this will overwrite the space of the extracted items.
+    """
+
+    def __init__(
+        self,
+        items: Iterable[Instance],
+        total: int | None = None,
+        limit: int | None = None,
+        overwrite_namespace: Namespace | None = None,
+    ) -> None:
+        self.items = items
+        self.total = total
+        self.limit = limit
+        self.overwrite_namespace = overwrite_namespace
+
+    @classmethod
+    def from_data_model(
+        cls, client: CogniteClient, data_model: DataModelIdentifier, limit: int | None = None
+    ) -> "DMSExtractor":
+        """Create an extractor from a data model.
+
+        Args:
+            client: The Cognite client to use.
+            data_model: The data model to extract.
+            limit: The maximum number of instances to extract.
+        """
+        retrieved = client.data_modeling.data_models.retrieve(data_model, inline_views=True)
+        if not retrieved:
+            raise ResourceRetrievalError(dm.DataModelId.load(data_model), "data model", "Data Model is missing in CDF")
+        return cls.from_views(client, retrieved.latest_version().views, limit)
+
+    @classmethod
+    def from_views(cls, client: CogniteClient, views: Iterable[dm.View], limit: int | None = None) -> "DMSExtractor":
+        """Create an extractor from a set of views.
+
+        Args:
+            client: The Cognite client to use.
+            views: The views to extract.
+            limit: The maximum number of instances to extract.
+        """
+        return cls(_InstanceIterator(client, views), total=None, limit=limit)
+
+    def extract(self) -> Iterable[Triple]:
+        for count, item in enumerate(self.items, 1):
+            if self.limit and count > self.limit:
+                break
+            yield from self._extract_instance(item)
+
+    def _extract_instance(self, instance: Instance) -> Iterable[Triple]:
+        if isinstance(instance, dm.Edge):
+            if not instance.properties:
+                yield (
+                    self._as_uri_ref(instance.start_node),
+                    self._as_uri_ref(instance.type),
+                    self._as_uri_ref(instance.end_node),
+                )
+                return
+            else:
+                # If the edge has properties, we create a node for the edge and connect it to the start and end nodes.
+                id_ = self._as_uri_ref(instance)
+                yield id_, RDF.type, self._as_uri_ref(instance.type)
+                yield id_, RDF.type, self._get_namespace(instance.space).Edge
+                yield (
+                    id_,
+                    self._as_uri_ref(dm.DirectRelationReference(instance.space, "startNode")),
+                    self._as_uri_ref(instance.start_node),
+                )
+                yield (
+                    id_,
+                    self._as_uri_ref(dm.DirectRelationReference(instance.space, "endNode")),
+                    self._as_uri_ref(instance.end_node),
+                )
+
+        elif isinstance(instance, dm.Node):
+            id_ = self._as_uri_ref(instance)
+            if instance.type:
+                type_ = self._as_uri_ref(cast(dm.DirectRelationReference, instance.type))
+            else:
+                type_ = self._get_namespace(instance.space).Node
+
+            yield id_, RDF.type, type_
+        else:
+            raise NotImplementedError(f"Unknown instance type {type(instance)}")
+
+        for view_id, properties in instance.properties.items():
+            namespace = self._get_namespace(view_id.space)
+            for key, value in properties.items():
+                for object_ in self._get_objects(value):
+                    yield id_, namespace[key], object_
+
+    def _get_objects(self, value: PropertyValue) -> Iterable[Literal | URIRef]:
+        if isinstance(value, str | float | bool | int):
+            yield Literal(value)
+        elif isinstance(value, dict) and "space" in value and "externalId" in value:
+            yield self._as_uri_ref(dm.DirectRelationReference.load(value))
+        elif isinstance(value, dict):
+            # This object is a json object.
+            yield Literal(str(value), datatype=XSD._NS["json"])
+        elif isinstance(value, list):
+            for item in value:
+                yield from self._get_objects(item)
+
+    def _as_uri_ref(self, instance: Instance | dm.DirectRelationReference) -> URIRef:
+        return self._get_namespace(instance.space)[instance.external_id]
+
+    def _get_namespace(self, space: str) -> Namespace:
+        if self.overwrite_namespace:
+            return self.overwrite_namespace
+        return Namespace(DEFAULT_SPACE_URI.format(space=space))
+
+
+class _InstanceIterator(Iterator[Instance]):
+    def __init__(self, client: CogniteClient, views: Iterable[dm.View]):
+        self.client = client
+        self.views = views
+
+    def __iter__(self) -> Iterator[Instance]:
+        return self
+
+    def __next__(self) -> Instance:  # type: ignore[misc]
+        for view in self.views:
+            # All nodes and edges with properties
+            yield from self.client.data_modeling.instances(chunk_size=None, instance_type="node", sources=[view])
+            yield from self.client.data_modeling.instances(chunk_size=None, instance_type="edge", sources=[view])
+
+            for prop in view.properties.values():
+                if isinstance(prop, dm.EdgeConnection):
+                    # Get all edges with properties
+                    yield from self.client.data_modeling.instances(
+                        chunk_size=None,
+                        instance_type="edge",
+                        filter=dm.filters.Equals(
+                            ["edge", "type"], {"space": prop.type.space, "externalId": prop.type.external_id}
+                        ),
+                    )

{cognite_neat-0.89.0 → cognite_neat-0.90.0}/cognite/neat/graph/extractors/_mock_graph_generator.py

@@ -17,7 +17,6 @@ from cognite.neat.rules.models import DMSRules, InformationRules
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
 from cognite.neat.rules.models.information import InformationProperty
-from cognite.neat.rules.transformers import DMSToInformation
 from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 from ._base import BaseExtractor
@@ -43,6 +42,9 @@ class MockGraphGenerator(BaseExtractor):
         allow_isolated_classes: bool = True,
     ):
         if isinstance(rules, DMSRules):
+            # fixes potential issues with circular dependencies
+            from cognite.neat.rules.transformers import DMSToInformation
+
             self.rules = DMSToInformation().transform(rules).rules
         elif isinstance(rules, InformationRules):
             self.rules = rules
@@ -152,21 +154,33 @@ def generate_triples(
     # generate triples for connected classes
     for class_ in generation_order:
         triples += _generate_triples_per_class(
-            class_, class_property_pairs, sym_pairs, instance_ids, namespace, stop_on_exception
+            class_,
+            class_property_pairs,
+            sym_pairs,
+            instance_ids,
+            namespace,
+            stop_on_exception,
         )
 
     # generate triples for isolated classes
     if allow_isolated_classes:
         for class_ in set(class_count.keys()) - set(generation_order):
             triples += _generate_triples_per_class(
-                class_, class_property_pairs, sym_pairs, instance_ids, namespace, stop_on_exception
+                class_,
+                class_property_pairs,
+                sym_pairs,
+                instance_ids,
+                namespace,
+                stop_on_exception,
             )
 
     return triples
 
 
 def _get_generation_order(
-    class_linkage: pd.DataFrame, parent_col: str = "source_class", child_col: str = "target_class"
+    class_linkage: pd.DataFrame,
+    parent_col: str = "source_class",
+    child_col: str = "target_class",
 ) -> dict:
     parent_child_list: list[list[str]] = class_linkage[[parent_col, child_col]].values.tolist()
     # Build a directed graph and a list of all names that have no parent
@@ -244,7 +258,10 @@ def _remove_non_requested_sym_pairs(class_linkage: pd.DataFrame, class_count: di
 
 
 def _generate_mock_data_property_triples(
-    instance_ids: list[URIRef], property_: str, namespace: Namespace, value_type: DataType
+    instance_ids: list[URIRef],
+    property_: str,
+    namespace: Namespace,
+    value_type: DataType,
 ) -> list[tuple[URIRef, URIRef, Literal]]:
     """Generates triples for data properties."""
 
@@ -254,7 +271,13 @@ def _generate_mock_data_property_triples(
         if python_type is int:
             triples.append((id_, URIRef(namespace[property_]), Literal(random.randint(1, 1983))))
         elif python_type is float:
-            triples.append((id_, URIRef(namespace[property_]), Literal(numpy.float32(random.uniform(1, 1983)))))
+            triples.append(
+                (
+                    id_,
+                    URIRef(namespace[property_]),
+                    Literal(numpy.float32(random.uniform(1, 1983))),
+                )
+            )
         # generate string
         else:
             triples.append(
@@ -293,7 +316,12 @@ def _generate_mock_object_property_triples(
 
     if tuple((class_, property_definition.value_type)) in sym_pairs:
         symmetric_class_properties = class_property_pairs[cast(ClassEntity, property_definition.value_type)]
-        candidates = list(filter(lambda instance: instance.value_type == class_, symmetric_class_properties))
+        candidates = list(
+            filter(
+                lambda instance: instance.value_type == class_,
+                symmetric_class_properties,
+            )
+        )
         symmetric_property = candidates[0]
         if len(candidates) > 1:
             warnings.warn(
@@ -310,10 +338,22 @@ def _generate_mock_object_property_triples(
         target = instance_ids[cast(ClassEntity, property_definition.value_type)][
             i % len(instance_ids[cast(ClassEntity, property_definition.value_type)])
         ]
-        triples += [(URIRef(source), URIRef(namespace[property_definition.property_]), URIRef(target))]
+        triples += [
+            (
+                URIRef(source),
+                URIRef(namespace[property_definition.property_]),
+                URIRef(target),
+            )
+        ]
 
         if symmetric_property:
-            triples += [(URIRef(target), URIRef(namespace[symmetric_property.property_]), URIRef(source))]
+            triples += [
+                (
+                    URIRef(target),
+                    URIRef(namespace[symmetric_property.property_]),
+                    URIRef(source),
+                )
+            ]
 
     if symmetric_property:
         class_property_pairs[cast(ClassEntity, property_definition.value_type)].remove(symmetric_property)

{cognite_neat-0.89.0 → cognite_neat-0.90.0}/cognite/neat/graph/loaders/_rdf2dms.py

@@ -13,8 +13,10 @@ from cognite.client.data_classes.data_modeling.ids import InstanceId
 from cognite.client.data_classes.data_modeling.views import SingleEdgeConnection
 from cognite.client.exceptions import CogniteAPIError
 from pydantic import BaseModel, ValidationInfo, create_model, field_validator
+from rdflib import RDF
 
 from cognite.neat.graph._tracking import LogTracker, Tracker
+from cognite.neat.graph.models import InstanceType
 from cognite.neat.issues import IssueList, NeatIssue, NeatIssueList
 from cognite.neat.issues.errors import (
     ResourceConvertionError,
@@ -108,7 +110,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         for view in self.data_model.views:
             view_id = view.as_id()
             tracker.start(repr(view_id))
-            pydantic_cls, edge_by_properties, issues = self._create_validation_classes(view)  # type: ignore[var-annotated]
+            pydantic_cls, edge_by_type, issues = self._create_validation_classes(view)  # type: ignore[var-annotated]
             yield from issues
             tracker.issue(issues)
             class_name = self.class_by_view_id.get(view.as_id(), view.external_id)
@@ -122,7 +124,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                     if stop_on_exception:
                         raise error from e
                     yield error
-                yield from self._create_edges(identifier, properties, edge_by_properties, tracker)
+                yield from self._create_edges(identifier, properties, edge_by_type, tracker)
             tracker.finish(repr(view_id))
 
     def write_to_file(self, filepath: Path) -> None:
@@ -147,16 +149,16 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
 
     def _create_validation_classes(
         self, view: dm.View
-    ) -> tuple[type[BaseModel], dict[str, dm.EdgeConnection], NeatIssueList]:
+    ) -> tuple[type[BaseModel], dict[str, tuple[str, dm.EdgeConnection]], NeatIssueList]:
         issues = IssueList()
         field_definitions: dict[str, tuple[type, Any]] = {}
-        edge_by_property: dict[str, dm.EdgeConnection] = {}
+        edge_by_property: dict[str, tuple[str, dm.EdgeConnection]] = {}
         validators: dict[str, classmethod] = {}
         direct_relation_by_property: dict[str, dm.DirectRelation] = {}
         json_fields: list[str] = []
         for prop_name, prop in view.properties.items():
             if isinstance(prop, dm.EdgeConnection):
-                edge_by_property[prop_name] = prop
+                edge_by_property[prop.type.external_id] = prop_name, prop
             if isinstance(prop, dm.MappedProperty):
                 if isinstance(prop.type, dm.DirectRelation):
                     direct_relation_by_property[prop_name] = prop.type
@@ -215,7 +217,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
 
             def parse_direct_relation(cls, value: list, info: ValidationInfo) -> dict | list[dict]:
                 # We validate above that we only get one value for single direct relations.
-                if cls.model_fields[info.field_name].annotation is list:
+                if list.__name__ in _get_field_value_types(cls, info):
                     return [{"space": self.instance_space, "externalId": v} for v in value]
                 elif value:
                     return {"space": self.instance_space, "externalId": value[0]}
@@ -231,16 +233,17 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
     def _create_node(
         self,
         identifier: str,
-        properties: dict[str, list[str]],
+        properties: dict[str | InstanceType, list[str]],
         pydantic_cls: type[BaseModel],
         view_id: dm.ViewId,
     ) -> dm.InstanceApply:
+        type_ = properties.pop(RDF.type, [None])[0]
         created = pydantic_cls.model_validate(properties)
 
         return dm.NodeApply(
             space=self.instance_space,
             external_id=identifier,
-            # type=#RDF type
+            type=dm.DirectRelationReference(view_id.space, type_) if type_ is not None else None,
             sources=[dm.NodeOrEdgeData(source=view_id, properties=dict(created.model_dump().items()))],
         )
 
@@ -248,13 +251,13 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         self,
         identifier: str,
         properties: dict[str, list[str]],
-        edge_by_properties: dict[str, dm.EdgeConnection],
+        edge_by_type: dict[str, tuple[str, dm.EdgeConnection]],
         tracker: Tracker,
     ) -> Iterable[dm.EdgeApply | NeatIssue]:
-        for prop, values in properties.items():
-            if prop not in edge_by_properties:
+        for predicate, values in properties.items():
+            if predicate not in edge_by_type:
                 continue
-            edge = edge_by_properties[prop]
+            prop_id, edge = edge_by_type[predicate]
             if isinstance(edge, SingleEdgeConnection) and len(values) > 1:
                 error = ResourceDuplicatedError(
                     resource_type="edge",
@@ -264,7 +267,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                 tracker.issue(error)
                 yield error
             for target in values:
-                external_id = f"{identifier}.{prop}.{target}"
+                external_id = f"{identifier}.{prop_id}.{target}"
                 yield dm.EdgeApply(
                     space=self.instance_space,
                     external_id=(external_id if len(external_id) < 256 else create_sha256_hash(external_id)),

{cognite_neat-0.89.0 → cognite_neat-0.90.0}/cognite/neat/graph/models.py

@@ -4,3 +4,4 @@ from rdflib import Literal
 from rdflib.term import URIRef
 
 Triple: TypeAlias = tuple[URIRef, URIRef, Literal | URIRef]
+InstanceType: TypeAlias = URIRef

{cognite_neat-0.89.0 → cognite_neat-0.90.0}/cognite/neat/graph/queries/_base.py

@@ -6,6 +6,7 @@ from rdflib import RDF, Graph, URIRef
 from rdflib import Literal as RdfLiteral
 from rdflib.query import ResultRow
 
+from cognite.neat.graph.models import InstanceType
 from cognite.neat.rules.models.entities import ClassEntity
 from cognite.neat.rules.models.information import InformationRules
 from cognite.neat.utils.rdf_ import remove_namespace_from_uri
@@ -98,7 +99,7 @@ class Queries:
         self,
         instance_id: URIRef,
         property_renaming_config: dict | None = None,
-    ) -> tuple[str, dict[str, list[str]]] | None:
+    ) -> tuple[str, dict[str | InstanceType, list[str]]] | None:
         """DESCRIBE instance for a given class from the graph store
 
         Args:
@@ -126,7 +127,8 @@ class Queries:
             # losing the namespace from the predicate!
             if not property_renaming_config and predicate != RDF.type:
                 property_values[remove_namespace_from_uri(predicate, validation="prefix")].append(value)
-
+            elif predicate == RDF.type:
+                property_values[RDF.type].append(value)
             # use-case: calling describe with renaming properties
             # renaming the property to the new name, if the property is defined
             # in the RULES sheet

{cognite_neat-0.89.0 → cognite_neat-0.90.0}/cognite/neat/issues/_base.py

@@ -45,6 +45,7 @@ ResourceType: TypeAlias = (
         "container property",
         "space",
         "class",
+        "property",
         "asset",
         "relationship",
         "data model",

{cognite_neat-0.89.0 → cognite_neat-0.90.0}/cognite/neat/issues/warnings/__init__.py

@@ -26,6 +26,7 @@ from ._properties import (
     PropertyDefinitionDuplicatedWarning,
     PropertyNotFoundWarning,
     PropertyTypeNotSupportedWarning,
+    PropertyValueTypeUndefinedWarning,
 )
 from ._resources import (
     ResourceNeatWarning,
@@ -49,6 +50,7 @@ __all__ = [
     "PropertyDefinitionDuplicatedWarning",
     "PropertyTypeNotSupportedWarning",
     "PropertyNotFoundWarning",
+    "PropertyValueTypeUndefinedWarning",
     "ResourceNeatWarning",
     "ResourcesDuplicatedWarning",
     "RegexViolationWarning",

{cognite_neat-0.89.0 → cognite_neat-0.90.0}/cognite/neat/issues/warnings/_properties.py

@@ -42,3 +42,15 @@ class PropertyDefinitionDuplicatedWarning(PropertyWarning[T_Identifier]):
     values: frozenset[str]
     default_action: str
     recommended_action: str | None = None
+
+
+@dataclass(frozen=True)
+class PropertyValueTypeUndefinedWarning(PropertyWarning[T_Identifier]):
+    """The {resource_type} with identifier {identifier} has a property {property_name}
+    which has undefined value type. This may result in unexpected behavior when exporting rules.
+    {default_action}"""
+
+    extra = "Recommended action: {recommended_action}"
+
+    default_action: str
+    recommended_action: str | None = None

{cognite_neat-0.89.0 → cognite_neat-0.90.0}/cognite/neat/rules/importers/_dms2rules.py

@@ -113,7 +113,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
                 DMSSchema(),
                 [
                     ResourceRetrievalError(
-                        dm.DataModelId.load(reference_model_id),  # type: ignore[arg-type]
+                        dm.DataModelId.load(data_model_id),  # type: ignore[arg-type]
                         "data model",
                         "Data Model is missing in CDF",
                     )