cognite-neat 0.90.1__tar.gz → 0.91.0__tar.gz

{cognite_neat-0.90.1 → cognite_neat-0.91.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.90.1
+Version: 0.91.0
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.91.0/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.91.0"

{cognite_neat-0.90.1 → cognite_neat-0.91.0}/cognite/neat/graph/extractors/__init__.py

@@ -8,6 +8,7 @@ from ._classic_cdf._sequences import SequencesExtractor
 from ._classic_cdf._timeseries import TimeSeriesExtractor
 from ._dexpi import DexpiExtractor
 from ._dms import DMSExtractor
+from ._iodd import IODDExtractor
 from ._mock_graph_generator import MockGraphGenerator
 from ._rdf_file import RdfFileExtractor
 
@@ -24,6 +25,7 @@ __all__ = [
     "RdfFileExtractor",
     "DexpiExtractor",
     "DMSExtractor",
+    "IODDExtractor",
 ]
 
 
@@ -38,6 +40,7 @@ TripleExtractors = (
     | LabelsExtractor
     | RdfFileExtractor
     | DexpiExtractor
+    | IODDExtractor
 )
 
 

{cognite_neat-0.90.1 → cognite_neat-0.91.0}/cognite/neat/graph/extractors/_dexpi.py

@@ -202,13 +202,13 @@ class DexpiExtractor(BaseExtractor):
 
     @classmethod
     def _get_element_label(cls, element: Element) -> str | None:
-        if children := get_children(element, "Label", 1):
-            if grandchildren := get_children(children[0], "Text", 1):
+        if children := get_children(element, "Label", no_children=1):
+            if grandchildren := get_children(children[0], "Text", no_children=1):
                 if "String" in grandchildren[0].attrib:
                     return grandchildren[0].attrib["String"]
 
         # extension for schema version 3.3, where text is used to "label" without a <label> parent
-        elif children := get_children(element, "Text", 1):
+        elif children := get_children(element, "Text", no_children=1):
             if "String" in children[0].attrib:
                 return children[0].attrib["String"]
 
@@ -219,7 +219,7 @@ class DexpiExtractor(BaseExtractor):
         # TODO: This requires more work as there are multiple groupings of GenericAttributes
 
         attributes = defaultdict(list)
-        if children := get_children(element, "GenericAttributes", 1):
+        if children := get_children(element, "GenericAttributes", no_children=1):
             if grandchildren := get_children(children[0], "GenericAttribute"):
                 for generic_attribute in grandchildren:
                     # extension for schema version 3.3, where "AttributeURI" is not included

cognite_neat-0.91.0/cognite/neat/graph/extractors/_iodd.py

@@ -0,0 +1,160 @@
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import ClassVar
+from xml.etree.ElementTree import Element
+
+from rdflib import RDF, Literal, Namespace, URIRef
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+from cognite.neat.utils.rdf_ import as_neat_compliant_uri
+from cognite.neat.utils.text import to_camel
+from cognite.neat.utils.xml_ import get_children
+
+IODD = Namespace("http://www.io-link.com/IODD/2010/10/")
+XSI = Namespace("http://www.w3.org/2001/XMLSchema-instance/")
+
+
+class IODDExtractor(BaseExtractor):
+    """
+    IODD-XML extractor of RDF triples
+
+    Each IODD sheet describes an IODD device. This extractor extracts rdf triples that describes the device, and the
+    sensors connected to the device.
+    This data is described under the elements "DeviceIdentity" and "ProcessDataCollection".
+    In addition, triples extacted from "DeviceIdentity" and
+    "ProcessDataCollection" may reference "Text" elements which are found under "ExternalTextCollection". Edges to
+    these Text element nodes are also extracted.
+
+    Args:
+        root: XML root element of IODD XML file.
+        namespace: Optional custom namespace to use for extracted triples that define data
+                    model instances. Defaults to DEFAULT_NAMESPACE.
+    """
+
+    device_elements_with_text_nodes: ClassVar[list[str]] = ["VendorText", "VendorUrl", "DeviceName", "DeviceFamily"]
+
+    def __init__(self, root: Element, namespace: Namespace | None = None):
+        self.root = root
+        self.namespace = namespace or DEFAULT_NAMESPACE
+
+    @classmethod
+    def from_file(cls, filepath: str | Path, namespace: Namespace | None = None):
+        return cls(ET.parse(filepath).getroot(), namespace)
+
+    @classmethod
+    def _from_root2triples(cls, root: Element, namespace: Namespace) -> list[Triple]:
+        """Loops through the relevant elements of the IODD XML sheet to create rdf triples that describes the IODD
+        device by starting at the root element.
+        """
+        triples: list[Triple] = []
+
+        # Extract DeviceIdentity triples
+        if di_root := get_children(
+            root, "DeviceIdentity", ignore_namespace=True, include_nested_children=True, no_children=1
+        ):
+            triples.extend(cls._iodd_device_identity2triples(di_root[0], namespace))
+
+        # Extract ProcessDataCollection triples -
+        # this element holds the information about the sensors with data coming from MQTT
+        if pc_root := get_children(
+            root, "ProcessDataCollection", ignore_namespace=True, include_nested_children=True, no_children=1
+        ):
+            triples.extend(cls._process_data_collection2triples(pc_root[0], namespace))
+
+        if et_root := get_children(
+            root, "ExternalTextCollection", ignore_namespace=True, include_nested_children=True, no_children=1
+        ):
+            triples.extend(cls._text_elements2triples(et_root[0], namespace))
+
+        return triples
+
+    @classmethod
+    def _device_2text_elements_edges(cls, di_root: Element, id: URIRef, namespace: Namespace) -> list[Triple]:
+        """
+        Create edges from the device node to text nodes.
+        """
+        triples: list[Triple] = []
+
+        for element_tag in cls.device_elements_with_text_nodes:
+            if child := get_children(
+                di_root, child_tag=element_tag, ignore_namespace=True, include_nested_children=True, no_children=1
+            ):
+                if text_id := child[0].attrib.get("textId"):
+                    # Create connection from device to textId node
+                    element_tag = to_camel(element_tag)
+                    triples.append((id, IODD[element_tag], namespace[text_id]))
+
+        return triples
+
+    @classmethod
+    def _text_elements2triples(cls, et_root: Element, namespace: Namespace) -> list[Triple]:
+        """
+        This method extracts all text item triples under the ExternalTextCollection element. This will create a node
+        for each text item, and add the text value as a property to the node.
+        """
+        triples: list[Triple] = []
+
+        if text_elements := get_children(
+            et_root, child_tag="Text", ignore_namespace=True, include_nested_children=True
+        ):
+            for element in text_elements:
+                if id := element.attrib.get("id"):
+                    text_id = namespace[id]
+
+                    # Create Text node
+                    triples.append((text_id, RDF.type, as_neat_compliant_uri(IODD["Text"])))
+
+                    # Resolve text value related to the text item
+                    if value := element.attrib.get("value"):
+                        triples.append((text_id, IODD.value, Literal(value)))
+
+        return triples
+
+    # TODO
+    @classmethod
+    def _process_data_collection2triples(cls, pc_root: Element, namespace: Namespace) -> list[Triple]:
+        """
+        ProcessDataCollection contains both ProcessDataIn and ProcessDataOut. Here, we are interested in the elements
+        that corresponds to actual sensor values that we will rececive from the MQTT connection.
+
+        Most likely this is ProcessDataOut elements (to be verified).
+        """
+        triples: list[Triple] = []
+
+        return triples
+
+    @classmethod
+    def _iodd_device_identity2triples(cls, di_root: Element, namespace: Namespace) -> list[Triple]:
+        """
+        Properties and metadata related to the IO Device are described under the 'DeviceIdentity' element in the XML.
+        This method extracts the triples that describe the device's identity which is found under the
+        DeviceIdentity element and its child elements.
+
+        """
+        triples: list[Triple] = []
+
+        if device_id := di_root.attrib.get("deviceId"):
+            id_ = namespace[device_id]
+
+            for attribute_name, attribute_value in di_root.attrib.items():
+                if attribute_name == "deviceId":
+                    # Create rdf type triple for IODD
+                    triples.append(
+                        (
+                            id_,
+                            RDF.type,
+                            as_neat_compliant_uri(IODD["IoddDevice"]),
+                        )
+                    )
+                else:
+                    # Collect attributes at root element
+                    triples.append((id_, IODD[attribute_name], Literal(attribute_value)))
+
+            triples.extend(cls._device_2text_elements_edges(di_root, id_, namespace))
+        return triples
+
+    def extract(self) -> list[Triple]:
+        # Extract triples from IODD device XML
+        return self._from_root2triples(self.root, self.namespace)

{cognite_neat-0.90.1 → cognite_neat-0.91.0}/cognite/neat/issues/_base.py

@@ -190,6 +190,10 @@ class NeatError(NeatIssue, Exception):
         read_info_by_sheet = kwargs.get("read_info_by_sheet")
 
         for error in errors:
+            if error["type"] == "is_instance_of" and error["loc"][1] == "is-instance[SheetList]":
+                # Skip the error for SheetList, as it is not relevant for the user. This is an
+                # internal class used to have helper methods for a lists as .to_pandas()
+                continue
             ctx = error.get("ctx")
             if isinstance(ctx, dict) and isinstance(multi_error := ctx.get("error"), MultiValueError):
                 if read_info_by_sheet:
@@ -363,10 +367,10 @@ class NeatIssueList(UserList[T_NeatIssue], Sequence[T_NeatIssue], ABC):
         """Return all the warnings in this list."""
         return type(self)([issue for issue in self if isinstance(issue, NeatWarning)])  # type: ignore[misc]
 
-    def as_errors(self) -> ExceptionGroup:
+    def as_errors(self, operation: str = "Operation failed") -> ExceptionGroup:
         """Return an ExceptionGroup with all the errors in this list."""
         return ExceptionGroup(
-            "Operation failed",
+            operation,
             [issue for issue in self if isinstance(issue, NeatError)],
         )
 

{cognite_neat-0.90.1 → cognite_neat-0.91.0}/cognite/neat/rules/exporters/_rules2excel.py

@@ -17,7 +17,7 @@ from cognite.neat.rules.models import (
     DataModelType,
     ExtensionCategory,
     SchemaCompleteness,
-    SheetEntity,
+    SheetRow,
 )
 from cognite.neat.rules.models.dms import DMSMetadata
 from cognite.neat.rules.models.domain import DomainMetadata
@@ -210,14 +210,14 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
                 cell.font = Font(bold=True, size=12)
 
     @classmethod
-    def _get_item_class(cls, annotation: GenericAlias) -> type[SheetEntity]:
+    def _get_item_class(cls, annotation: GenericAlias) -> type[SheetRow]:
         if not isinstance(annotation, GenericAlias):
             raise ValueError(f"Expected annotation to be a GenericAlias, but got {type(annotation)}")
         args = get_args(annotation)
         if len(args) != 1:
             raise ValueError(f"Expected annotation to have exactly one argument, but got {len(args)}")
         arg = args[0]
-        if not issubclass(arg, SheetEntity):
+        if not issubclass(arg, SheetRow):
             raise ValueError(f"Expected annotation to have a BaseModel argument, but got {type(arg)}")
         return arg
 

{cognite_neat-0.90.1 → cognite_neat-0.91.0}/cognite/neat/rules/exporters/_rules2yaml.py

@@ -58,6 +58,10 @@ class YAMLExporter(BaseExporter[VerifiedRules, str]):
     def export(self, rules: VerifiedRules) -> str:
         """Export rules to YAML (or JSON) format.
 
+        This will export the rules to YAML format if the output is set to "yaml" and JSON format if the output is set.
+        All None and Unset values are excluded from the output to keep the output clean, i.e., only the values the user
+        has set.
+
         Args:
             rules: The rules to be exported.
 
@@ -66,7 +70,7 @@ class YAMLExporter(BaseExporter[VerifiedRules, str]):
         """
         # model_dump_json ensures that the output is in JSON format,
         # if we don't do this, we will get Enums and other types that are not serializable to YAML
-        json_output = rules.dump(mode="json")
+        json_output = rules.dump(mode="json", exclude_none=True, exclude_unset=True)
         if self.output == "json":
             return json.dumps(json_output)
         elif self.output == "yaml":

{cognite_neat-0.90.1 → cognite_neat-0.91.0}/cognite/neat/rules/models/__init__.py

@@ -4,7 +4,7 @@ from cognite.neat.rules.models.domain import DomainInputRules, DomainRules
 from cognite.neat.rules.models.information._rules import InformationRules
 from cognite.neat.rules.models.information._rules_input import InformationInputRules
 
-from ._base_rules import DataModelType, ExtensionCategory, RoleTypes, SchemaCompleteness, SheetEntity, SheetList
+from ._base_rules import DataModelType, ExtensionCategory, RoleTypes, SchemaCompleteness, SheetList, SheetRow
 from .dms._rules import DMSRules
 from .dms._rules_input import DMSInputRules
 from .dms._schema import DMSSchema
@@ -42,5 +42,5 @@ __all__ = [
     "ExtensionCategory",
     "DataModelType",
     "SheetList",
-    "SheetEntity",
+    "SheetRow",
 ]

{cognite_neat-0.90.1 → cognite_neat-0.91.0}/cognite/neat/rules/models/_base_input.py

@@ -18,7 +18,7 @@ from dataclasses import Field, dataclass, fields, is_dataclass
 from types import GenericAlias, UnionType
 from typing import Any, Generic, TypeVar, Union, cast, get_args, get_origin, overload
 
-from ._base_rules import BaseRules, RuleModel
+from ._base_rules import BaseRules, SchemaModel
 
 if sys.version_info >= (3, 11):
     from typing import Self
@@ -26,7 +26,7 @@ else:
     from typing_extensions import Self
 
 T_BaseRules = TypeVar("T_BaseRules", bound=BaseRules)
-T_RuleModel = TypeVar("T_RuleModel", bound=RuleModel)
+T_RuleModel = TypeVar("T_RuleModel", bound=SchemaModel)
 
 
 @dataclass