cognite-neat 0.81.12__tar.gz → 0.82.1__tar.gz

{cognite_neat-0.81.12 → cognite_neat-0.82.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.81.12
+Version: 0.82.1
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.82.1/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.82.1"

{cognite_neat-0.81.12 → cognite_neat-0.82.1}/cognite/neat/graph/extractors/_mock_graph_generator.py

@@ -2,7 +2,6 @@
 It is a bit ugly and needs some proper refactoring, but it is not a priority at the moment.
 """
 
-import logging
 import random
 import warnings
 from collections import OrderedDict
@@ -87,11 +86,11 @@ def generate_triples(
     stop_on_exception: bool = False,
     allow_isolated_classes: bool = True,
 ) -> list[Triple]:
-    """Generate mock triples based on data model defined transformation rules and desired number
+    """Generate mock triples based on data model defined in rules and desired number
     of class instances
 
     Args:
-        transformation_rules : Transformation rules defining the data model
+        rules : Rules defining the data model
         class_count: Target class count for each class in the ontology
         stop_on_exception: To stop if exception is encountered or not, default is False
         allow_isolated_classes: To allow generation of instances for classes that are not
@@ -107,11 +106,9 @@ def generate_triples(
     if non_existing_classes := set(class_count.keys()) - defined_classes:
         msg = f"Class count contains classes {non_existing_classes} for which properties are not defined in Data Model!"
         if stop_on_exception:
-            logging.error(msg)
             raise ValueError(msg)
         else:
             msg += " These classes will be ignored."
-            logging.warning(msg)
             warnings.warn(msg, stacklevel=2)
             for class_ in non_existing_classes:
                 class_count.pop(class_)
@@ -279,14 +276,12 @@ def _generate_mock_object_property_triples(
     if property_definition.value_type not in instance_ids:
         msg = f"Class {property_definition.value_type} not found in class count! "
         if stop_on_exception:
-            logging.error(msg)
             raise ValueError(msg)
         else:
             msg += (
                 f"Skipping creating triples for property {property_definition.name} "
                 f"of class {class_.suffix} which expects values of this type!"
             )
-            logging.warning(msg)
             warnings.warn(msg, stacklevel=2)
             return []
 
@@ -354,7 +349,6 @@ def _generate_triples_per_class(
             )
 
         else:
-            logging.error(f"Property type {property_.value_type} not supported!")
             raise ValueError(f"Property type {property_.value_type} not supported!")
 
     return triples

{cognite_neat-0.81.12 → cognite_neat-0.82.1}/cognite/neat/graph/loaders/_rdf2dms.py

@@ -93,7 +93,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
             yield from issues
             tracker.issue(issues)
             class_name = self.class_by_view_id.get(view.as_id(), view.external_id)
-            triples = self.graph_store.queries.triples_of_type_instances(class_name)
+            triples = self.graph_store.read(class_name)
             for identifier, properties in _triples2dictionary(triples).items():
                 try:
                     yield self._create_node(identifier, properties, pydantic_cls, view_id)

cognite_neat-0.82.1/cognite/neat/graph/queries/__init__.py

@@ -0,0 +1,3 @@
+from ._base import Queries
+
+__all__ = ["Queries"]

cognite_neat-0.82.1/cognite/neat/graph/queries/_base.py

@@ -0,0 +1,99 @@
+import warnings
+from typing import cast
+
+from rdflib import RDF, Graph, URIRef
+from rdflib.query import ResultRow
+
+from cognite.neat.rules.models.entities import ClassEntity
+from cognite.neat.rules.models.information import InformationRules
+from cognite.neat.utils.utils import remove_namespace
+
+from ._construct import build_construct_query
+
+
+class Queries:
+    """Helper class for storing standard queries for the graph store."""
+
+    def __init__(self, graph: Graph, rules: InformationRules | None = None):
+        self.graph = graph
+        self.rules = rules
+
+    def list_instances_ids_of_class(self, class_uri: URIRef, limit: int = -1) -> list[URIRef]:
+        """Get instances ids for a given class
+
+        Args:
+            class_uri: Class for which instances are to be found
+            limit: Max number of instances to return, by default -1 meaning all instances
+
+        Returns:
+            List of class instance URIs
+        """
+        query_statement = "SELECT DISTINCT ?subject WHERE { ?subject a <class> .} LIMIT X".replace(
+            "class", class_uri
+        ).replace("LIMIT X", "" if limit == -1 else f"LIMIT {limit}")
+        return [cast(tuple, res)[0] for res in list(self.graph.query(query_statement))]
+
+    def list_instances_of_type(self, class_uri: URIRef) -> list[ResultRow]:
+        """Get all triples for instances of a given class
+
+        Args:
+            class_uri: Class for which instances are to be found
+
+        Returns:
+            List of triples for instances of the given class
+        """
+        query = (
+            f"SELECT ?instance ?prop ?value "
+            f"WHERE {{ ?instance rdf:type <{class_uri}> . ?instance ?prop ?value . }} order by ?instance "
+        )
+
+        # Select queries gives an iterable of result rows
+        return cast(list[ResultRow], list(self.graph.query(query)))
+
+    def triples_of_type_instances(self, rdf_type: str) -> list[tuple[str, str, str]]:
+        """Get all triples of a given type.
+
+        This method assumes the graph has been transformed into the default namespace.
+        """
+
+        if self.rules:
+            query = (
+                f"SELECT ?instance ?prop ?value "
+                f"WHERE {{ ?instance a <{self.rules.metadata.namespace[rdf_type]}> . ?instance ?prop ?value . }} "
+                "order by ?instance"
+            )
+
+            result = self.graph.query(query)
+
+            # We cannot include the RDF.type in case there is a neat:type property
+            return [remove_namespace(*triple) for triple in result if triple[1] != RDF.type]  # type: ignore[misc, index]
+        else:
+            warnings.warn("No rules found for the graph store, returning empty list.", stacklevel=2)
+            return []
+
+    def construct_instances_of_class(self, class_: str, properties_optional: bool = True) -> list[tuple[str, str, str]]:
+        """CONSTRUCT instances for a given class from the graph store
+
+        Args:
+            class_: Class entity for which we want to generate query
+            properties_optional: Whether to make all properties optional, default True
+
+        Returns:
+            List of triples for instances of the given class
+        """
+
+        if self.rules and (
+            query := build_construct_query(
+                ClassEntity(prefix=self.rules.metadata.prefix, suffix=class_),
+                self.graph,
+                self.rules,
+                properties_optional,
+            )
+        ):
+            result = self.graph.query(query)
+
+            # We cannot include the RDF.type in case there is a neat:type property
+            return [remove_namespace(*triple) for triple in result if triple[1] != RDF.type]  # type: ignore[misc, index]
+        else:
+            warnings.warn("No rules found for the graph store, returning empty list.", stacklevel=2)
+            return []

cognite_neat-0.82.1/cognite/neat/graph/queries/_construct.py

@@ -0,0 +1,185 @@
+import re
+from typing import cast
+
+from rdflib import Graph, URIRef
+
+from cognite.neat.rules.analysis import InformationArchitectRulesAnalysis
+from cognite.neat.rules.models._rdfpath import (
+    AllReferences,
+    Hop,
+    RDFPath,
+    SingleProperty,
+    Traversal,
+)
+from cognite.neat.rules.models.entities import ClassEntity
+from cognite.neat.rules.models.information import InformationProperty, InformationRules
+from cognite.neat.utils.utils import most_occurring_element
+
+from ._shared import Triple, hop2property_path
+
+_QUERY_TEMPLATE = """CONSTRUCT {{ {graph_template} }}
+                     WHERE {{     {graph_pattern}
+                                  {filter}
+                     }}"""
+
+
+def build_construct_query(
+    class_: ClassEntity,
+    graph: Graph,
+    rules: InformationRules,
+    properties_optional: bool = True,
+    class_instances: list[URIRef] | None = None,
+) -> str | None:
+    """Builds a CONSTRUCT query for a given class and rules and optionally filters by class instances.
+
+    Args:
+        class_ : The class entity for which the query is generated.
+        graph : The graph containing instances of classes.
+        rules : The information rules to use for query generation.
+        properties_optional : Whether to make all properties optional. Defaults to True.
+        class_instances : List of class instances to filter by. Defaults to None (no filter, return all instances).
+
+    Returns:
+        str: CONSTRUCT query.
+
+    !!! note "On CONSTRUCT Query"
+        CONSTRUCT query is composed of two parts: graph template and graph pattern.
+        Graph template is used the shape of instance acquired using graph pattern.
+        This allows us to create a new graph with the new shape without actually modifying
+        the original graph, or creating new instances.
+
+        The CONSTRUCT query is far less forgiving than the SELECT query. It will not return
+        anything if one of the properties that define the "shape" of the class instance is missing.
+        This is the reason why there is an option to make all properties optional, so that
+        the query will return all instances that have at least one property defined.
+    """
+    if (
+        transformations := InformationArchitectRulesAnalysis(rules)
+        .class_property_pairs(only_rdfpath=True, consider_inheritance=True)
+        .get(class_, None)
+    ):
+        templates, patterns = to_construct_triples(
+            graph, list(transformations.values()), rules.prefixes, properties_optional
+        )
+
+        return _QUERY_TEMPLATE.format(
+            graph_template="\n".join(triples2sparql_statement(templates)),
+            graph_pattern="\n".join(triples2sparql_statement(patterns)),
+            filter="" if not class_instances else add_filter(class_instances),
+        )
+
+    else:
+        return None
+
+
+def add_filter(class_instances: list[URIRef]):
+    class_instances_formatted = [f"<{instance}>" for instance in class_instances]
+    return f"FILTER (?instance IN ({', '.join(class_instances_formatted)}))"
+
+
+def to_construct_triples(
+    graph: Graph, transformations: list[InformationProperty], prefixes: dict, properties_optional: bool = True
+) -> tuple[list[Triple], list[Triple]]:
+    """Converts transformations of a class to CONSTRUCT triples which are used to generate CONSTRUCT query
+
+    Args:
+        graph: Graph containing instances of classes (used for property inference for hops)
+        transformations : List of transformations to use to form triples
+        prefixes : Dictionary of prefixes for namespaces
+        properties_optional : Flag indicating if properties should be optional. Defaults to True.
+
+    Returns:
+        tuple: Tuple of triples that define graph template and graph pattern parts of CONSTRUCT query
+
+
+    !!! note "Purely inherited transformations"
+        Assumption that neat makes is that in case of purely inherited transformations
+        we will type instance with class to which transformation belongs to.
+
+        Otherwise we will type instance with class that is most occurring in non-inherited
+        transformations.
+
+    """
+    # TODO: Add handling of UNIONs in rules
+
+    templates = []
+    patterns = []
+    non_inherited_starting_rdf_types = []
+
+    for transformation in transformations:
+        traversal = cast(RDFPath, transformation.transformation).traversal
+
+        # keeping track of starting rdf types of non-inherited transformations/properties
+        if isinstance(traversal, Traversal) and not transformation.inherited:
+            non_inherited_starting_rdf_types.append(traversal.class_.id)
+
+        graph_template_triple = Triple(
+            subject="?instance",
+            predicate=f"{transformation.class_.prefix}:{transformation.property_}",
+            object=f'?{re.sub(r"[^_a-zA-Z0-9/_]", "_", str(transformation.property_).lower())}',
+            optional=False,
+        )
+        templates.append(graph_template_triple)
+
+        # use case AllReferences: binding instance to certain rdf property
+        if isinstance(traversal, AllReferences):
+            graph_pattern_triple = Triple(
+                subject="BIND(?instance", predicate="AS", object=f"{graph_template_triple.object})", optional=False
+            )
+
+        # use case SingleProperty: simple property traversal
+        elif isinstance(traversal, SingleProperty):
+            graph_pattern_triple = Triple(
+                subject=graph_template_triple.subject,
+                predicate=traversal.property.id,
+                object=graph_template_triple.object,
+                optional=True if properties_optional else not transformation.is_mandatory,
+            )
+
+        # use case Hop: property traversal with multiple hops turned into property path
+        # see: https://www.oxfordsemantic.tech/faqs/what-is-a-property-path
+        elif isinstance(traversal, Hop):
+            graph_pattern_triple = Triple(
+                subject="?instance",
+                predicate=hop2property_path(graph, traversal, prefixes),
+                object=graph_template_triple.object,
+                optional=True if properties_optional else not transformation.is_mandatory,
+            )
+
+        # other type of rdfpaths are skipped
+        else:
+            continue
+
+        patterns.append(graph_pattern_triple)
+
+    # Add first triple for graph pattern stating type of object
+    # we use most occurring here to pull out most occurring rdf type of the starting
+    # node of the transformation, or the class itself to which the transformation is
+    # defined for.
+    # This is safeguard in case there are multiple classes in the graph pattern
+    patterns.insert(
+        0,
+        Triple(
+            subject="?instance",
+            predicate="a",
+            object=(
+                most_occurring_element(non_inherited_starting_rdf_types)
+                if non_inherited_starting_rdf_types
+                else str(transformation.class_)
+            ),
+            optional=False,
+        ),
+    )
+
+    return templates, patterns
+
+
+def triples2sparql_statement(triples: list[Triple]):
+    return [
+        (
+            f"OPTIONAL {{ {triple.subject} {triple.predicate} {triple.object} . }}"
+            if triple.optional
+            else f"{triple.subject} {triple.predicate} {triple.object} ."
+        )
+        for triple in triples
+    ]

cognite_neat-0.82.1/cognite/neat/graph/queries/_shared.py

@@ -0,0 +1,159 @@
+import sys
+from collections.abc import Iterable
+from typing import ClassVar, cast
+
+from pydantic import BaseModel, ConfigDict, Field
+from rdflib import Graph, Literal, Namespace
+from rdflib.term import URIRef
+
+from cognite.neat.constants import PREFIXES
+from cognite.neat.rules.models._rdfpath import (
+    Hop,
+    Step,
+)
+from cognite.neat.utils.utils import remove_namespace, uri_to_short_form
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class Triple(BaseModel):
+    model_config: ClassVar[ConfigDict] = ConfigDict(
+        populate_by_name=True, arbitrary_types_allowed=True, strict=False, extra="allow"
+    )
+
+    subject: str | URIRef
+    predicate: str | URIRef
+    object: str | URIRef | Literal | None = None
+    optional: bool = Field(
+        description="Indicates whether a triple is optional, used when building SPARQL query",
+        default=False,
+    )
+
+    @classmethod
+    def from_rdflib_triple(cls, triple: tuple[URIRef, URIRef, URIRef | Literal]) -> Self:
+        return cls(subject=triple[0], predicate=triple[1], object=triple[2])
+
+
+def generate_prefix_header(prefixes: dict[str, Namespace] = PREFIXES) -> str:
+    """Generate prefix header which is added to SPARQL query and allows for shorten query statements
+
+    Parameters
+    ----------
+    prefixes : dict
+        Dict containing prefix - namespace pairs, default PREFIXES
+
+    Returns
+    -------
+    str
+        Prefix header
+    """
+    return "".join(f"PREFIX {key}:<{value}>\n" for key, value in prefixes.items())
+
+
+def get_predicate_id(
+    graph: Graph, subject_type_id: str, object_type_id: str, prefixes: dict[str, Namespace] = PREFIXES
+) -> URIRef:
+    """Returns predicate (aka property) URI (i.e., ID) that connects subject and object
+
+    Parameters
+    ----------
+    graph : Graph
+        Data model graph or data model instance (aka knowledge graph)
+    subject_type_id : str
+        ID of subject type (aka subject class)
+    object_type_id : str
+        ID of object type (aka object class)
+    prefixes : dict, optional
+        Dict containing prefix - namespace pairs, default PREFIXES
+
+    Returns
+    -------
+    URIRef
+        ID of predicate (aka property) connecting subject and object
+    """
+    query = """
+
+        SELECT ?predicateTypeID
+        WHERE {
+            ?subjectInstanceID a subjectTypeID .
+            ?objectInstanceID a objectTypeID .
+            ?subjectInstanceID ?predicateTypeID ?objectInstanceID .
+        } LIMIT 1"""
+
+    query = query.replace("insertPrefixes", generate_prefix_header(prefixes))
+    final_query = query.replace("subjectTypeID", subject_type_id).replace("objectTypeID", object_type_id)
+    res = list(cast(tuple, graph.query(final_query)))
+
+    if len(res) != 1:
+        raise ValueError("Subject and Object must have exactly 1 relation!")
+
+    return res[0][0]
+
+
+def hop2property_path(graph: Graph, hop: Hop, prefixes: dict[str, Namespace]) -> str:
+    """Converts hop to property path string
+
+    Parameters
+    ----------
+    graph : Graph
+        Graph containing instances of classes
+    hop : Hop
+        Hop to convert
+    prefixes : dict[str, Namespace]
+        Dictionary of prefixes to use for compression and predicate querying
+
+    Returns
+    -------
+    str
+        Property path string for hop traversal (e.g. ^rdf:type/rdfs:subClassOf)
+    """
+
+    # setting previous step to origin, as we are starting from there
+    previous_step = Step(class_=hop.class_, direction="origin")
+
+    # add triples for all steps until destination
+    property_path = ""
+    for current_step in hop.traversal:
+        sub_entity, obj_entity = (
+            (current_step, previous_step) if current_step.direction == "source" else (previous_step, current_step)
+        )
+
+        predicate_raw = get_predicate_id(graph, sub_entity.class_.id, obj_entity.class_.id, prefixes)
+
+        predicate = uri_to_short_form(predicate_raw, prefixes)
+
+        predicate = f"^{predicate}" if current_step.direction == "source" else predicate
+        property_path += f"{predicate}/"
+
+        previous_step = current_step
+
+    if previous_step.property:
+        return property_path + previous_step.property.id
+    else:
+        # removing "/" at the end of property path if there is no property at the end
+        return property_path[:-1]
+
+
+def triples2dictionary(triples: Iterable[tuple[URIRef, URIRef, str | URIRef]]) -> dict[URIRef, dict[str, list[str]]]:
+    """Converts list of triples to dictionary"""
+    dictionary: dict[URIRef, dict[str, list[str]]] = {}
+    for triple in triples:
+        id_: str
+        property_: str
+        value: str
+        uri: URIRef
+
+        id_, property_, value = remove_namespace(*triple)  # type: ignore[misc]
+        uri = triple[0]
+
+        if uri not in dictionary:
+            dictionary[uri] = {"external_id": [id_]}
+
+        if property_ not in dictionary[uri]:
+            dictionary[uri][property_] = [value]
+        else:
+            dictionary[uri][property_].append(value)
+    return dictionary

{cognite_neat-0.81.12 → cognite_neat-0.82.1}/cognite/neat/graph/stores/_base.py

@@ -5,18 +5,17 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import cast
 
-from rdflib import RDF, Graph, Namespace, URIRef
+from rdflib import Graph, Namespace, URIRef
 from rdflib.plugins.stores.sparqlstore import SPARQLUpdateStore
-from rdflib.query import ResultRow
 
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph._shared import MIMETypes
 from cognite.neat.graph.extractors import RdfFileExtractor, TripleExtractors
 from cognite.neat.graph.models import Triple
+from cognite.neat.graph.queries import Queries
 from cognite.neat.graph.transformers import Transformers
 from cognite.neat.rules.models.entities import ClassEntity
 from cognite.neat.rules.models.information import InformationRules
-from cognite.neat.utils import remove_namespace
 from cognite.neat.utils.auxiliary import local_import
 
 from ._provenance import Change, Provenance
@@ -59,32 +58,30 @@ class NeatGraphStore:
         )
 
         if rules:
-            self.rules = rules
-            self.base_namespace = self.rules.metadata.namespace
-            self.provenance.append(
-                Change.record(
-                    activity=f"{type(self)}.rules",
-                    start=_start,
-                    end=datetime.now(timezone.utc),
-                    description=f"Added rules to graph store as {type(self.rules).__name__}",
-                )
-            )
-
-            if self.rules.prefixes:
-                self._upsert_prefixes(self.rules.prefixes)
-                self.provenance.append(
-                    Change.record(
-                        activity=f"{type(self).__name__}._upsert_prefixes",
-                        start=_start,
-                        end=datetime.now(timezone.utc),
-                        description="Upsert prefixes to graph store",
-                    )
-                )
-
+            self.add_rules(rules)
         else:
             self.base_namespace = DEFAULT_NAMESPACE
 
-        self.queries = _Queries(self)
+        self.queries = Queries(self.graph, self.rules)
+
+    def add_rules(self, rules: InformationRules) -> None:
+        """This method is used to add rules to the graph store and it is the only correct
+        way to add rules to the graph store, after the graph store has been initialized."""
+
+        self.rules = rules
+        self.base_namespace = self.rules.metadata.namespace
+        self.queries = Queries(self.graph, self.rules)
+        self.provenance.append(
+            Change.record(
+                activity=f"{type(self)}.rules",
+                start=datetime.now(timezone.utc),
+                end=datetime.now(timezone.utc),
+                description=f"Added rules to graph store as {type(self.rules).__name__}",
+            )
+        )
+
+        if self.rules.prefixes:
+            self._upsert_prefixes(self.rules.prefixes)
 
     def _upsert_prefixes(self, prefixes: dict[str, Namespace]) -> None:
         """Adds prefixes to the graph store."""
@@ -181,7 +178,7 @@ class NeatGraphStore:
             warnings.warn("Desired type not found in graph!", stacklevel=2)
             return []
 
-        return []
+        return self.queries.construct_instances_of_class(class_)
 
     def _parse_file(
         self,
@@ -281,63 +278,3 @@ class NeatGraphStore:
                     description=transformer.description,
                 )
             )
-
-
-class _Queries:
-    """Helper class for storing standard queries for the graph store."""
-
-    def __init__(self, store: NeatGraphStore):
-        self.store = store
-
-    def list_instances_ids_of_class(self, class_uri: URIRef, limit: int = -1) -> list[URIRef]:
-        """Get instances ids for a given class
-
-        Args:
-            class_uri: Class for which instances are to be found
-            limit: Max number of instances to return, by default -1 meaning all instances
-
-        Returns:
-            List of class instance URIs
-        """
-        query_statement = "SELECT DISTINCT ?subject WHERE { ?subject a <class> .} LIMIT X".replace(
-            "class", class_uri
-        ).replace("LIMIT X", "" if limit == -1 else f"LIMIT {limit}")
-        return [cast(tuple, res)[0] for res in list(self.store.graph.query(query_statement))]
-
-    def list_instances_of_type(self, class_uri: URIRef) -> list[ResultRow]:
-        """Get all triples for instances of a given class
-
-        Args:
-            class_uri: Class for which instances are to be found
-
-        Returns:
-            List of triples for instances of the given class
-        """
-        query = (
-            f"SELECT ?instance ?prop ?value "
-            f"WHERE {{ ?instance rdf:type <{class_uri}> . ?instance ?prop ?value . }} order by ?instance "
-        )
-
-        # Select queries gives an iterable of result rows
-        return cast(list[ResultRow], list(self.store.graph.query(query)))
-
-    def triples_of_type_instances(self, rdf_type: str) -> list[tuple[str, str, str]]:
-        """Get all triples of a given type.
-
-        This method assumes the graph has been transformed into the default namespace.
-        """
-
-        if self.store.rules:
-            query = (
-                f"SELECT ?instance ?prop ?value "
-                f"WHERE {{ ?instance a <{self.store.rules.metadata.namespace[rdf_type]}> . ?instance ?prop ?value . }} "
-                "order by ?instance"
-            )
-
-            result = self.store.graph.query(query)
-
-            # We cannot include the RDF.type in case there is a neat:type property
-            return [remove_namespace(*triple) for triple in result if triple[1] != RDF.type]  # type: ignore[misc, index]
-        else:
-            warnings.warn("No rules found for the graph store, returning empty list.", stacklevel=2)
-            return []

cognite_neat-0.82.1/cognite/neat/rules/_shared.py

@@ -0,0 +1,5 @@
+from typing import TypeAlias
+
+from cognite.neat.rules.models import AssetRules, DMSRules, DomainRules, InformationRules
+
+Rules: TypeAlias = DomainRules | InformationRules | DMSRules | AssetRules