cognite-neat 0.80.1__tar.gz → 0.81.0__tar.gz

{cognite_neat-0.80.1 → cognite_neat-0.81.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.80.1
+Version: 0.81.0
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0
@@ -43,7 +43,7 @@ Requires-Dist: prometheus-client (>=0.20.0,<0.21.0)
 Requires-Dist: pydantic (>=2,<3)
 Requires-Dist: pymdown-extensions ; extra == "docs"
 Requires-Dist: pyoxigraph (==0.3.19)
-Requires-Dist: python-multipart (>=0.0.6,<0.0.7)
+Requires-Dist: python-multipart (>=0,<1)
 Requires-Dist: rdflib
 Requires-Dist: requests
 Requires-Dist: schedule (>=1,<2)

cognite_neat-0.81.0/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.81.0"

{cognite_neat-0.80.1 → cognite_neat-0.81.0}/cognite/neat/graph/extractors/__init__.py

@@ -5,6 +5,7 @@ from ._classic_cdf._labels import LabelsExtractor
 from ._classic_cdf._relationships import RelationshipsExtractor
 from ._classic_cdf._sequences import SequencesExtractor
 from ._classic_cdf._timeseries import TimeSeriesExtractor
+from ._dexpi import DexpiExtractor
 from ._mock_graph_generator import MockGraphGenerator
 from ._rdf_file import RdfFileExtractor
 
@@ -18,6 +19,7 @@ __all__ = [
     "FilesExtractor",
     "LabelsExtractor",
     "RdfFileExtractor",
+    "DexpiExtractor",
 ]
 
 
@@ -31,4 +33,5 @@ TripleExtractors = (
     | FilesExtractor
     | LabelsExtractor
     | RdfFileExtractor
+    | DexpiExtractor
 )

cognite_neat-0.81.0/cognite/neat/graph/extractors/_dexpi.py

@@ -0,0 +1,212 @@
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+from collections.abc import Iterable
+from pathlib import Path
+from xml.etree.ElementTree import Element
+
+from rdflib import RDF, RDFS, XSD, Literal, Namespace, URIRef
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+from cognite.neat.utils.utils import as_neat_compliant_uri
+from cognite.neat.utils.xml import get_children, iterate_tree
+
+DEXPI = Namespace("http://sandbox.dexpi.org/rdl/")
+
+
+class DexpiExtractor(BaseExtractor):
+    """
+    DEXPI-XML extractor of RDF triples
+
+    Args:
+        root: XML root element of DEXPI file.
+        namespace: Optional custom namespace to use for extracted triples that define data
+                    model instances. Defaults to DEFAULT_NAMESPACE.
+    """
+
+    def __init__(
+        self,
+        root: Element,
+        namespace: Namespace | None = None,
+    ):
+        self.root = root
+        self.namespace = namespace or DEFAULT_NAMESPACE
+
+    @classmethod
+    def from_file(cls, filepath: str | Path, namespace: Namespace | None = None):
+        return cls(ET.parse(filepath).getroot(), namespace)
+
+    @classmethod
+    def from_url(cls, url: str, namespace: Namespace | None = None):
+        from io import BytesIO
+
+        import requests
+
+        response = requests.get(url)
+        response.raise_for_status()
+        return cls(ET.parse(BytesIO(response.content)).getroot(), namespace)
+
+    def extract(self) -> Iterable[Triple]:
+        """Extracts RDF triples from DEXPI XML file."""
+
+        for element in iterate_tree(self.root):
+            yield from self._element2triples(element, self.namespace)
+
+    @classmethod
+    def _element2triples(cls, element: Element, namespace: Namespace) -> list[Triple]:
+        """Converts an element to triples."""
+        triples: list[Triple] = []
+
+        if (
+            "ComponentClass" in element.attrib
+            and element.attrib["ComponentClass"] != "Label"
+            and "ID" in element.attrib
+        ):
+            id_ = namespace[element.attrib["ID"]]
+
+            if node_triples := cls._element2node_triples(id_, element):
+                triples.extend(node_triples)
+
+            if edge_triples := cls._element2edge_triples(id_, element, namespace):
+                triples.extend(edge_triples)
+
+        return triples
+
+    @classmethod
+    def _element2edge_triples(cls, id_: URIRef, element: Element, namespace: Namespace) -> list[Triple]:
+        triples: list[Triple] = []
+
+        # connection triples
+        if connections := get_children(element, "Connection"):
+            for connection in connections:
+                if "FromID" in connection.attrib and "ToID" in connection.attrib:
+                    triples.append(
+                        (
+                            namespace[connection.attrib["FromID"]],
+                            DEXPI.connection,
+                            namespace[connection.attrib["ToID"]],
+                        )
+                    )
+
+        # association triples
+        if associations := get_children(element, "Association"):
+            for association in associations:
+                if "Type" in association.attrib and "ItemID" in association.attrib:
+                    association_type = cls._to_uri_friendly_association_type(association)
+
+                    triples.append(
+                        (
+                            id_,
+                            DEXPI[f"association/{association_type}"],
+                            namespace[association.attrib["ItemID"]],
+                        )
+                    )
+
+        # children-parent triples
+        for child in element:
+            if "ID" in child.attrib and child.tag != "Label":
+                camel_case_property = child.tag[0].lower() + child.tag[1:]
+                triples.append(
+                    (
+                        id_,
+                        DEXPI[f"children/{camel_case_property}"],
+                        namespace[child.attrib["ID"]],
+                    )
+                )
+
+        return triples
+
+    @classmethod
+    def _to_uri_friendly_association_type(cls, association: Element):
+        association_type = "".join(
+            [word.capitalize() if i != 0 else word for i, word in enumerate(association.attrib["Type"].split(" "))]
+        )
+
+        return association_type
+
+    @classmethod
+    def _element2node_triples(cls, id_: URIRef, element: Element) -> list[Triple]:
+        """Converts an XML element to triples."""
+        triples: list[Triple] = []
+
+        # adding tag triple if exists
+        if tag := element.tag:
+            triples.append((id_, DEXPI.tag, Literal(str(tag))))
+
+        # adding attributes triples
+        if attributes := element.attrib:
+            if component_class := attributes.get("ComponentClass", None):
+                triples.append((id_, DEXPI.ComponentClass, Literal(component_class)))
+            if component_name := attributes.get("ComponentName", None):
+                triples.append((id_, DEXPI.ComponentName, Literal(component_name)))
+            if type_ := attributes.get("ComponentClassURI", None):
+                triples.append((id_, RDF.type, URIRef(type_)))
+
+        # add label triple
+        if label := cls._get_element_label(element):
+            triples.append((id_, RDFS.label, Literal(label)))
+
+        # add generic attributes triples
+        if generic_attributes := cls._get_element_generic_attributes(element):
+            for attribute, value_definitions in generic_attributes.items():
+                predicate = as_neat_compliant_uri(attribute)
+                for value_definition in value_definitions:
+                    if literal := cls._value_definition2literal(value_definition):
+                        triples.append((id_, predicate, literal))
+
+        return triples
+
+    @classmethod
+    def _value_definition2literal(cls, definition: dict) -> Literal | None:
+        if "Value" not in definition or "Format" not in definition:
+            return None
+
+        # case: when language is present we create add language tag to the literal
+        elif "Language" in definition and "Value" in definition:
+            return Literal(definition["Value"], lang=definition["Language"])
+
+        # case: when ValueURI is present we use it instead of Value
+        # this would be candidate for ENUMs in CDF
+        elif "ValueURI" in definition:
+            return Literal(definition["ValueURI"], datatype=XSD[definition["Format"]])
+
+        # case: when Format is not string we make sure to add the datatype
+        elif definition["Format"].lower() != "string":
+            return Literal(definition["Value"], datatype=XSD[definition["Format"]])
+
+        # case: when Format is string we add the literal without datatype (easier to read triples, less noise)
+        else:
+            return Literal(definition["Value"])
+
+    @classmethod
+    def _get_element_label(cls, element: Element) -> str | None:
+        if children := get_children(element, "Label", 1):
+            if grandchildren := get_children(children[0], "Text", 1):
+                if "String" in grandchildren[0].attrib:
+                    return grandchildren[0].attrib["String"]
+
+        # extension for schema version 3.3, where text is used to "label" without a <label> parent
+        elif children := get_children(element, "Text", 1):
+            if "String" in children[0].attrib:
+                return children[0].attrib["String"]
+
+        return None
+
+    @classmethod
+    def _get_element_generic_attributes(cls, element: Element) -> dict:
+        # TODO: This requires more work as there are multiple groupings of GenericAttributes
+
+        attributes = defaultdict(list)
+        if children := get_children(element, "GenericAttributes", 1):
+            if grandchildren := get_children(children[0], "GenericAttribute"):
+                for generic_attribute in grandchildren:
+                    # extension for schema version 3.3, where "AttributeURI" is not included
+                    if "AttributeURI" in generic_attribute.attrib:
+                        if generic_attribute.attrib["AttributeURI"] not in attributes:
+                            attributes[generic_attribute.attrib["AttributeURI"]] = [generic_attribute.attrib]
+
+                        else:
+                            attributes[generic_attribute.attrib["AttributeURI"]].append(generic_attribute.attrib)
+
+        return attributes

{cognite_neat-0.80.1 → cognite_neat-0.81.0}/cognite/neat/graph/stores/_base.py

@@ -1,11 +1,10 @@
 import sys
 import warnings
 from collections.abc import Iterable
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import cast
 
-import pytz
 from rdflib import RDF, Graph, Namespace, URIRef
 from rdflib.plugins.stores.sparqlstore import SPARQLUpdateStore
 from rdflib.query import ResultRow
@@ -41,14 +40,14 @@ class NeatGraphStore:
         graph: Graph,
         rules: InformationRules | None = None,
     ):
-        _start = datetime.now(pytz.utc)
+        _start = datetime.now(timezone.utc)
         self.graph = graph
         self.provenance = Provenance(
             [
                 Change.record(
                     activity=f"{type(self).__name__}.__init__",
                     start=_start,
-                    end=datetime.now(pytz.utc),
+                    end=datetime.now(timezone.utc),
                     description=f"Initialize graph store as {type(self.graph.store).__name__}",
                 )
             ]
@@ -62,7 +61,7 @@ class NeatGraphStore:
 
     def _upsert_prefixes(self, prefixes: dict[str, Namespace]) -> None:
         """Adds prefixes to the graph store."""
-        _start = datetime.now(pytz.utc)
+        _start = datetime.now(timezone.utc)
         for prefix, namespace in prefixes.items():
             self.graph.bind(prefix, namespace)
 
@@ -70,7 +69,7 @@ class NeatGraphStore:
             Change.record(
                 activity=f"{type(self).__name__}._upsert_prefixes",
                 start=_start,
-                end=datetime.now(pytz.utc),
+                end=datetime.now(timezone.utc),
                 description="Upsert prefixes to graph store",
             )
         )
@@ -124,11 +123,22 @@ class NeatGraphStore:
         return cls(graph, rules)
 
     def write(self, extractor: TripleExtractors) -> None:
+        _start = datetime.now(timezone.utc)
+
         if isinstance(extractor, RdfFileExtractor):
             self._parse_file(extractor.filepath, extractor.mime_type, extractor.base_uri)
         else:
             self._add_triples(extractor.extract())
 
+        self.provenance.append(
+            Change.record(
+                activity=f"{type(extractor).__name__}",
+                start=_start,
+                end=datetime.now(timezone.utc),
+                description=f"Extracted triples to graph store using {type(extractor).__name__}",
+            )
+        )
+
     def _parse_file(
         self,
         filepath: Path,

{cognite_neat-0.80.1 → cognite_neat-0.81.0}/cognite/neat/utils/utils.py

@@ -147,6 +147,13 @@ def get_namespace(URI: URIRef, special_separator: str = "#_") -> str:
         return "/".join(URI.split("/")[:-1]) + "/"
 
 
+def as_neat_compliant_uri(uri: URIRef) -> URIRef:
+    namespace = get_namespace(uri)
+    id_ = remove_namespace(uri)
+    compliant_uri = re.sub(r"[^a-zA-Z0-9-_.]", "", id_)
+    return URIRef(f"{namespace}{compliant_uri}")
+
+
 def convert_rdflib_content(content: Literal | URIRef | dict | list) -> Any:
     if isinstance(content, Literal) or isinstance(content, URIRef):
         return content.toPython()

{cognite_neat-0.80.1 → cognite_neat-0.81.0}/cognite/neat/workflows/steps/lib/current/rules_importer.py

@@ -178,7 +178,7 @@ class DMSToRules(Step):
         ),
         Configurable(
             name="Reference data model id",
-            value=None,
+            value="",
             label="The ID of the Reference Data Model to import. Written at 'my_space:my_data_model(version=1)'. "
             "This is typically an enterprise data model when you want to import a solution model",
             type="string",

{cognite_neat-0.80.1 → cognite_neat-0.81.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.80.1"
+version = "0.81.0"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [
@@ -77,7 +77,7 @@ cognite-sdk = "^7.37.0"
 deepdiff = "*"
 fastapi = "^0"
 schedule = "^1"
-python-multipart = "^0.0.6"
+python-multipart = "^0"
 oxrdflib = { version = "^0.3.3", optional = true, extras = ["oxigraph"] }
 
 

cognite_neat-0.80.1/cognite/neat/_version.py

@@ -1 +0,0 @@
-__version__ = "0.80.1"