cognite-neat 0.90.2__py3-none-any.whl → 0.92.0__py3-none-any.whl

cognite/neat/graph/extractors/_classic_cdf/_relationships.py

@@ -1,7 +1,7 @@
-from collections.abc import Callable, Set
+from collections import defaultdict
+from collections.abc import Callable, Iterable, Set
 from datetime import datetime, timezone
 from pathlib import Path
-from urllib.parse import quote
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes import Relationship, RelationshipList
@@ -10,10 +10,11 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.graph.models import Triple
 from cognite.neat.utils.auxiliary import create_sha256_hash
 
-from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFExtractor
+from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFBaseExtractor, InstanceIdPrefix
+from ._labels import LabelsExtractor
 
 
-class RelationshipsExtractor(ClassicCDFExtractor[Relationship]):
+class RelationshipsExtractor(ClassicCDFBaseExtractor[Relationship]):
     """Extract data from Cognite Data Fusions Relationships into Neat.
 
     Args:
@@ -34,6 +35,31 @@ class RelationshipsExtractor(ClassicCDFExtractor[Relationship]):
 
     _default_rdf_type = "Relationship"
 
+    def __init__(
+        self,
+        items: Iterable[Relationship],
+        namespace: Namespace | None = None,
+        to_type: Callable[[Relationship], str | None] | None = None,
+        total: int | None = None,
+        limit: int | None = None,
+        unpack_metadata: bool = True,
+        skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
+    ):
+        super().__init__(
+            items,
+            namespace=namespace,
+            to_type=to_type,
+            total=total,
+            limit=limit,
+            unpack_metadata=unpack_metadata,
+            skip_metadata_values=skip_metadata_values,
+        )
+        # This is used by the ClassicExtractor to log the target nodes, such
+        # that it can extract them.
+        # It is private to avoid exposing it to the user.
+        self._log_target_nodes = False
+        self._target_external_ids_by_type: dict[InstanceIdPrefix, set[str]] = defaultdict(set)
+
     @classmethod
     def from_dataset(
         cls,
@@ -79,8 +105,13 @@ class RelationshipsExtractor(ClassicCDFExtractor[Relationship]):
         """Converts an asset to triples."""
 
         if relationship.external_id and relationship.source_external_id and relationship.target_external_id:
+            if self._log_target_nodes and relationship.target_type and relationship.target_external_id:
+                self._target_external_ids_by_type[InstanceIdPrefix.from_str(relationship.target_type)].add(
+                    relationship.target_external_id
+                )
+
             # relationships do not have an internal id, so we generate one
-            id_ = self.namespace[f"Relationship_{create_sha256_hash(relationship.external_id)}"]
+            id_ = self.namespace[f"{InstanceIdPrefix.relationship}{create_sha256_hash(relationship.external_id)}"]
 
             type_ = self._get_rdf_type(relationship)
             # Set rdf type
@@ -178,7 +209,7 @@ class RelationshipsExtractor(ClassicCDFExtractor[Relationship]):
                         (
                             id_,
                             self.namespace.label,
-                            self.namespace[f"Label_{quote(label.dump()['externalId'])}"],
+                            self.namespace[f"{InstanceIdPrefix.label}{LabelsExtractor._label_id(label)}"],
                         )
                     )
 
@@ -188,7 +219,7 @@ class RelationshipsExtractor(ClassicCDFExtractor[Relationship]):
                     (
                         id_,
                         self.namespace.dataset,
-                        self.namespace[f"Dataset_{relationship.data_set_id}"],
+                        self.namespace[f"{InstanceIdPrefix.data_set}{relationship.data_set_id}"],
                     )
                 )
 

cognite/neat/graph/extractors/_classic_cdf/_sequences.py

@@ -8,10 +8,10 @@ from rdflib import RDF, Literal, Namespace
 
 from cognite.neat.graph.models import Triple
 
-from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFExtractor
+from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFBaseExtractor, InstanceIdPrefix
 
 
-class SequencesExtractor(ClassicCDFExtractor[Sequence]):
+class SequencesExtractor(ClassicCDFBaseExtractor[Sequence]):
     """Extract data from Cognite Data Fusions Sequences into Neat.
 
     Args:
@@ -56,6 +56,31 @@ class SequencesExtractor(ClassicCDFExtractor[Sequence]):
             skip_metadata_values=skip_metadata_values,
         )
 
+    @classmethod
+    def from_hierarchy(
+        cls,
+        client: CogniteClient,
+        root_asset_external_id: str,
+        namespace: Namespace | None = None,
+        to_type: Callable[[Sequence], str | None] | None = None,
+        limit: int | None = None,
+        unpack_metadata: bool = True,
+        skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
+    ):
+        total = client.sequences.aggregate_count(
+            filter=SequenceFilter(asset_subtree_ids=[{"externalId": root_asset_external_id}])
+        )
+
+        return cls(
+            client.sequences(asset_subtree_external_ids=[root_asset_external_id]),
+            namespace,
+            to_type,
+            total,
+            limit,
+            unpack_metadata=unpack_metadata,
+            skip_metadata_values=skip_metadata_values,
+        )
+
     @classmethod
     def from_file(
         cls,
@@ -78,7 +103,7 @@ class SequencesExtractor(ClassicCDFExtractor[Sequence]):
         )
 
     def _item2triples(self, sequence: Sequence) -> list[Triple]:
-        id_ = self.namespace[f"Sequence_{sequence.id}"]
+        id_ = self.namespace[f"{InstanceIdPrefix.sequence}{sequence.id}"]
 
         type_ = self._get_rdf_type(sequence)
         # Set rdf type
@@ -121,7 +146,7 @@ class SequencesExtractor(ClassicCDFExtractor[Sequence]):
                 (
                     id_,
                     self.namespace.data_set_id,
-                    self.namespace[f"Dataset_{sequence.data_set_id}"],
+                    self.namespace[f"{InstanceIdPrefix.data_set}{sequence.data_set_id}"],
                 )
             )
 
@@ -130,7 +155,7 @@ class SequencesExtractor(ClassicCDFExtractor[Sequence]):
                 (
                     id_,
                     self.namespace.asset,
-                    self.namespace[f"Asset_{sequence.asset_id}"],
+                    self.namespace[f"{InstanceIdPrefix.asset}{sequence.asset_id}"],
                 )
             )
 

cognite/neat/graph/extractors/_classic_cdf/_timeseries.py

@@ -9,10 +9,10 @@ from rdflib import RDF, Literal, Namespace, URIRef
 
 from cognite.neat.graph.models import Triple
 
-from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFExtractor
+from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFBaseExtractor, InstanceIdPrefix
 
 
-class TimeSeriesExtractor(ClassicCDFExtractor[TimeSeries]):
+class TimeSeriesExtractor(ClassicCDFBaseExtractor[TimeSeries]):
     """Extract data from Cognite Data Fusions TimeSeries into Neat.
 
     Args:
@@ -58,6 +58,31 @@ class TimeSeriesExtractor(ClassicCDFExtractor[TimeSeries]):
             skip_metadata_values=skip_metadata_values,
         )
 
+    @classmethod
+    def from_hierarchy(
+        cls,
+        client: CogniteClient,
+        root_asset_external_id: str,
+        namespace: Namespace | None = None,
+        to_type: Callable[[TimeSeries], str | None] | None = None,
+        limit: int | None = None,
+        unpack_metadata: bool = True,
+        skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
+    ):
+        total = client.time_series.aggregate_count(
+            filter=TimeSeriesFilter(asset_subtree_ids=[{"externalId": root_asset_external_id}])
+        )
+
+        return cls(
+            client.time_series(asset_external_ids=[root_asset_external_id]),
+            namespace,
+            to_type,
+            total,
+            limit,
+            unpack_metadata=unpack_metadata,
+            skip_metadata_values=skip_metadata_values,
+        )
+
     @classmethod
     def from_file(
         cls,
@@ -80,7 +105,7 @@ class TimeSeriesExtractor(ClassicCDFExtractor[TimeSeries]):
         )
 
     def _item2triples(self, timeseries: TimeSeries) -> list[Triple]:
-        id_ = self.namespace[f"TimeSeries_{timeseries.id}"]
+        id_ = self.namespace[f"{InstanceIdPrefix.time_series}{timeseries.id}"]
 
         # Set rdf type
         type_ = self._get_rdf_type(timeseries)
@@ -158,7 +183,7 @@ class TimeSeriesExtractor(ClassicCDFExtractor[TimeSeries]):
                 (
                     id_,
                     self.namespace.dataset,
-                    self.namespace[f"Dataset_{timeseries.data_set_id}"],
+                    self.namespace[f"{InstanceIdPrefix.data_set}{timeseries.data_set_id}"],
                 )
             )
 
@@ -167,7 +192,7 @@ class TimeSeriesExtractor(ClassicCDFExtractor[TimeSeries]):
                 (
                     id_,
                     self.namespace.asset,
-                    self.namespace[f"Asset_{timeseries.asset_id}"],
+                    self.namespace[f"{InstanceIdPrefix.asset}{timeseries.asset_id}"],
                 )
             )
 

cognite/neat/graph/extractors/_dexpi.py

@@ -202,13 +202,13 @@ class DexpiExtractor(BaseExtractor):
 
     @classmethod
     def _get_element_label(cls, element: Element) -> str | None:
-        if children := get_children(element, "Label", 1):
-            if grandchildren := get_children(children[0], "Text", 1):
+        if children := get_children(element, "Label", no_children=1):
+            if grandchildren := get_children(children[0], "Text", no_children=1):
                 if "String" in grandchildren[0].attrib:
                     return grandchildren[0].attrib["String"]
 
         # extension for schema version 3.3, where text is used to "label" without a <label> parent
-        elif children := get_children(element, "Text", 1):
+        elif children := get_children(element, "Text", no_children=1):
             if "String" in children[0].attrib:
                 return children[0].attrib["String"]
 
@@ -219,7 +219,7 @@ class DexpiExtractor(BaseExtractor):
         # TODO: This requires more work as there are multiple groupings of GenericAttributes
 
         attributes = defaultdict(list)
-        if children := get_children(element, "GenericAttributes", 1):
+        if children := get_children(element, "GenericAttributes", no_children=1):
             if grandchildren := get_children(children[0], "GenericAttribute"):
                 for generic_attribute in grandchildren:
                     # extension for schema version 3.3, where "AttributeURI" is not included

cognite/neat/graph/extractors/_iodd.py

@@ -0,0 +1,160 @@
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import ClassVar
+from xml.etree.ElementTree import Element
+
+from rdflib import RDF, Literal, Namespace, URIRef
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+from cognite.neat.utils.rdf_ import as_neat_compliant_uri
+from cognite.neat.utils.text import to_camel
+from cognite.neat.utils.xml_ import get_children
+
+IODD = Namespace("http://www.io-link.com/IODD/2010/10/")
+XSI = Namespace("http://www.w3.org/2001/XMLSchema-instance/")
+
+
+class IODDExtractor(BaseExtractor):
+    """
+    IODD-XML extractor of RDF triples
+
+    Each IODD sheet describes an IODD device. This extractor extracts rdf triples that describes the device, and the
+    sensors connected to the device.
+    This data is described under the elements "DeviceIdentity" and "ProcessDataCollection".
+    In addition, triples extacted from "DeviceIdentity" and
+    "ProcessDataCollection" may reference "Text" elements which are found under "ExternalTextCollection". Edges to
+    these Text element nodes are also extracted.
+
+    Args:
+        root: XML root element of IODD XML file.
+        namespace: Optional custom namespace to use for extracted triples that define data
+                    model instances. Defaults to DEFAULT_NAMESPACE.
+    """
+
+    device_elements_with_text_nodes: ClassVar[list[str]] = ["VendorText", "VendorUrl", "DeviceName", "DeviceFamily"]
+
+    def __init__(self, root: Element, namespace: Namespace | None = None):
+        self.root = root
+        self.namespace = namespace or DEFAULT_NAMESPACE
+
+    @classmethod
+    def from_file(cls, filepath: str | Path, namespace: Namespace | None = None):
+        return cls(ET.parse(filepath).getroot(), namespace)
+
+    @classmethod
+    def _from_root2triples(cls, root: Element, namespace: Namespace) -> list[Triple]:
+        """Loops through the relevant elements of the IODD XML sheet to create rdf triples that describes the IODD
+        device by starting at the root element.
+        """
+        triples: list[Triple] = []
+
+        # Extract DeviceIdentity triples
+        if di_root := get_children(
+            root, "DeviceIdentity", ignore_namespace=True, include_nested_children=True, no_children=1
+        ):
+            triples.extend(cls._iodd_device_identity2triples(di_root[0], namespace))
+
+        # Extract ProcessDataCollection triples -
+        # this element holds the information about the sensors with data coming from MQTT
+        if pc_root := get_children(
+            root, "ProcessDataCollection", ignore_namespace=True, include_nested_children=True, no_children=1
+        ):
+            triples.extend(cls._process_data_collection2triples(pc_root[0], namespace))
+
+        if et_root := get_children(
+            root, "ExternalTextCollection", ignore_namespace=True, include_nested_children=True, no_children=1
+        ):
+            triples.extend(cls._text_elements2triples(et_root[0], namespace))
+
+        return triples
+
+    @classmethod
+    def _device_2text_elements_edges(cls, di_root: Element, id: URIRef, namespace: Namespace) -> list[Triple]:
+        """
+        Create edges from the device node to text nodes.
+        """
+        triples: list[Triple] = []
+
+        for element_tag in cls.device_elements_with_text_nodes:
+            if child := get_children(
+                di_root, child_tag=element_tag, ignore_namespace=True, include_nested_children=True, no_children=1
+            ):
+                if text_id := child[0].attrib.get("textId"):
+                    # Create connection from device to textId node
+                    element_tag = to_camel(element_tag)
+                    triples.append((id, IODD[element_tag], namespace[text_id]))
+
+        return triples
+
+    @classmethod
+    def _text_elements2triples(cls, et_root: Element, namespace: Namespace) -> list[Triple]:
+        """
+        This method extracts all text item triples under the ExternalTextCollection element. This will create a node
+        for each text item, and add the text value as a property to the node.
+        """
+        triples: list[Triple] = []
+
+        if text_elements := get_children(
+            et_root, child_tag="Text", ignore_namespace=True, include_nested_children=True
+        ):
+            for element in text_elements:
+                if id := element.attrib.get("id"):
+                    text_id = namespace[id]
+
+                    # Create Text node
+                    triples.append((text_id, RDF.type, as_neat_compliant_uri(IODD["Text"])))
+
+                    # Resolve text value related to the text item
+                    if value := element.attrib.get("value"):
+                        triples.append((text_id, IODD.value, Literal(value)))
+
+        return triples
+
+    # TODO
+    @classmethod
+    def _process_data_collection2triples(cls, pc_root: Element, namespace: Namespace) -> list[Triple]:
+        """
+        ProcessDataCollection contains both ProcessDataIn and ProcessDataOut. Here, we are interested in the elements
+        that corresponds to actual sensor values that we will rececive from the MQTT connection.
+
+        Most likely this is ProcessDataOut elements (to be verified).
+        """
+        triples: list[Triple] = []
+
+        return triples
+
+    @classmethod
+    def _iodd_device_identity2triples(cls, di_root: Element, namespace: Namespace) -> list[Triple]:
+        """
+        Properties and metadata related to the IO Device are described under the 'DeviceIdentity' element in the XML.
+        This method extracts the triples that describe the device's identity which is found under the
+        DeviceIdentity element and its child elements.
+
+        """
+        triples: list[Triple] = []
+
+        if device_id := di_root.attrib.get("deviceId"):
+            id_ = namespace[device_id]
+
+            for attribute_name, attribute_value in di_root.attrib.items():
+                if attribute_name == "deviceId":
+                    # Create rdf type triple for IODD
+                    triples.append(
+                        (
+                            id_,
+                            RDF.type,
+                            as_neat_compliant_uri(IODD["IoddDevice"]),
+                        )
+                    )
+                else:
+                    # Collect attributes at root element
+                    triples.append((id_, IODD[attribute_name], Literal(attribute_value)))
+
+            triples.extend(cls._device_2text_elements_edges(di_root, id_, namespace))
+        return triples
+
+    def extract(self) -> list[Triple]:
+        # Extract triples from IODD device XML
+        return self._from_root2triples(self.root, self.namespace)

cognite/neat/issues/_base.py

@@ -190,6 +190,10 @@ class NeatError(NeatIssue, Exception):
         read_info_by_sheet = kwargs.get("read_info_by_sheet")
 
         for error in errors:
+            if error["type"] == "is_instance_of" and error["loc"][1] == "is-instance[SheetList]":
+                # Skip the error for SheetList, as it is not relevant for the user. This is an
+                # internal class used to have helper methods for a lists as .to_pandas()
+                continue
             ctx = error.get("ctx")
             if isinstance(ctx, dict) and isinstance(multi_error := ctx.get("error"), MultiValueError):
                 if read_info_by_sheet:
@@ -363,10 +367,10 @@ class NeatIssueList(UserList[T_NeatIssue], Sequence[T_NeatIssue], ABC):
         """Return all the warnings in this list."""
         return type(self)([issue for issue in self if isinstance(issue, NeatWarning)])  # type: ignore[misc]
 
-    def as_errors(self) -> ExceptionGroup:
+    def as_errors(self, operation: str = "Operation failed") -> ExceptionGroup:
         """Return an ExceptionGroup with all the errors in this list."""
         return ExceptionGroup(
-            "Operation failed",
+            operation,
             [issue for issue in self if isinstance(issue, NeatError)],
         )
 

cognite/neat/rules/exporters/_rules2excel.py

@@ -17,7 +17,7 @@ from cognite.neat.rules.models import (
     DataModelType,
     ExtensionCategory,
     SchemaCompleteness,
-    SheetEntity,
+    SheetRow,
 )
 from cognite.neat.rules.models.dms import DMSMetadata
 from cognite.neat.rules.models.domain import DomainMetadata
@@ -210,14 +210,14 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
                 cell.font = Font(bold=True, size=12)
 
     @classmethod
-    def _get_item_class(cls, annotation: GenericAlias) -> type[SheetEntity]:
+    def _get_item_class(cls, annotation: GenericAlias) -> type[SheetRow]:
         if not isinstance(annotation, GenericAlias):
             raise ValueError(f"Expected annotation to be a GenericAlias, but got {type(annotation)}")
         args = get_args(annotation)
         if len(args) != 1:
             raise ValueError(f"Expected annotation to have exactly one argument, but got {len(args)}")
         arg = args[0]
-        if not issubclass(arg, SheetEntity):
+        if not issubclass(arg, SheetRow):
             raise ValueError(f"Expected annotation to have a BaseModel argument, but got {type(arg)}")
         return arg
 

cognite/neat/rules/exporters/_rules2yaml.py

@@ -58,6 +58,10 @@ class YAMLExporter(BaseExporter[VerifiedRules, str]):
     def export(self, rules: VerifiedRules) -> str:
         """Export rules to YAML (or JSON) format.
 
+        This will export the rules to YAML format if the output is set to "yaml" and JSON format if the output is set.
+        All None and Unset values are excluded from the output to keep the output clean, i.e., only the values the user
+        has set.
+
         Args:
             rules: The rules to be exported.
 
@@ -66,7 +70,7 @@ class YAMLExporter(BaseExporter[VerifiedRules, str]):
         """
         # model_dump_json ensures that the output is in JSON format,
         # if we don't do this, we will get Enums and other types that are not serializable to YAML
-        json_output = rules.dump(mode="json")
+        json_output = rules.dump(mode="json", exclude_none=True, exclude_unset=True)
         if self.output == "json":
             return json.dumps(json_output)
         elif self.output == "yaml":

cognite/neat/rules/models/__init__.py

@@ -4,7 +4,7 @@ from cognite.neat.rules.models.domain import DomainInputRules, DomainRules
 from cognite.neat.rules.models.information._rules import InformationRules
 from cognite.neat.rules.models.information._rules_input import InformationInputRules
 
-from ._base_rules import DataModelType, ExtensionCategory, RoleTypes, SchemaCompleteness, SheetEntity, SheetList
+from ._base_rules import DataModelType, ExtensionCategory, RoleTypes, SchemaCompleteness, SheetList, SheetRow
 from .dms._rules import DMSRules
 from .dms._rules_input import DMSInputRules
 from .dms._schema import DMSSchema
@@ -42,5 +42,5 @@ __all__ = [
     "ExtensionCategory",
     "DataModelType",
     "SheetList",
-    "SheetEntity",
+    "SheetRow",
 ]

cognite/neat/rules/models/_base_input.py

@@ -18,7 +18,7 @@ from dataclasses import Field, dataclass, fields, is_dataclass
 from types import GenericAlias, UnionType
 from typing import Any, Generic, TypeVar, Union, cast, get_args, get_origin, overload
 
-from ._base_rules import BaseRules, RuleModel
+from ._base_rules import BaseRules, SchemaModel
 
 if sys.version_info >= (3, 11):
     from typing import Self
@@ -26,7 +26,7 @@ else:
     from typing_extensions import Self
 
 T_BaseRules = TypeVar("T_BaseRules", bound=BaseRules)
-T_RuleModel = TypeVar("T_RuleModel", bound=RuleModel)
+T_RuleModel = TypeVar("T_RuleModel", bound=SchemaModel)
 
 
 @dataclass