cognite-neat 0.85.12__py3-none-any.whl → 0.87.0__py3-none-any.whl

cognite/neat/graph/stores/_base.py

@@ -15,6 +15,7 @@ from cognite.neat.graph.extractors import RdfFileExtractor, TripleExtractors
 from cognite.neat.graph.models import Triple
 from cognite.neat.graph.queries import Queries
 from cognite.neat.graph.transformers import Transformers
+from cognite.neat.rules.analysis import InformationAnalysis
 from cognite.neat.rules.models import InformationRules
 from cognite.neat.rules.models.entities import ClassEntity
 from cognite.neat.utils.auxiliary import local_import
@@ -165,25 +166,33 @@ class NeatGraphStore:
             )
         )
 
-    def read(self, class_: str) -> list[tuple[str, str, str]]:
+    def read(self, class_: str) -> Iterable[tuple[str, dict[str, list[str]]]]:
         """Read instances for given view from the graph store."""
-        # PLACEHOLDER: Implement reading instances for a given view
-        # not yet developed
 
         if not self.rules:
-            warnings.warn(
-                "No rules found for the graph store, returning empty list.",
-                stacklevel=2,
-            )
-            return []
+            warnings.warn("Rules not found in graph store!", stacklevel=2)
+            return None
 
         class_entity = ClassEntity(prefix=self.rules.metadata.prefix, suffix=class_)
 
         if class_entity not in [definition.class_ for definition in self.rules.classes.data]:
             warnings.warn("Desired type not found in graph!", stacklevel=2)
-            return []
+            return None
+
+        if InformationAnalysis(self.rules).has_hop_transformations():
+            warnings.warn(
+                "Rules contain Hop rdfpath, run ReduceHopTraversal transformer first!",
+                stacklevel=2,
+            )
+            return None
+
+        instance_ids = self.queries.list_instances_ids_of_class(self.rules.metadata.namespace[class_])
+
+        property_renaming_config = InformationAnalysis(self.rules).define_property_renaming_config(class_entity)
+        print(property_renaming_config)
 
-        return self.queries.construct_instances_of_class(class_)
+        for instance_id in instance_ids:
+            yield self.queries.describe(instance_id, property_renaming_config)
 
     def _parse_file(
         self,

cognite/neat/graph/transformers/_rdfpath.py

@@ -0,0 +1,7 @@
+from ._base import BaseTransformer
+
+
+class ReduceHopTraversal(BaseTransformer):
+    """ReduceHopTraversal is a transformer that reduces the number of hops to direct connection."""
+
+    ...

cognite/neat/legacy/graph/extractors/_dexpi.py

@@ -9,11 +9,6 @@ from cognite.neat.legacy.graph.models import Triple
 
 from ._base import BaseExtractor
 
-_DEXPI_PREFIXES = {
-    "dexpi": Namespace("http://sandbox.dexpi.org/rdl/"),
-    "posccaesar": Namespace("http://data.posccaesar.org/rdl/"),
-}
-
 
 class DexpiXML(BaseExtractor):
     """

cognite/neat/legacy/graph/stores/_base.py

@@ -11,7 +11,7 @@ from prometheus_client import Gauge, Summary
 from rdflib import Graph, Namespace, URIRef
 from rdflib.query import Result, ResultRow
 
-from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
+from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
 from cognite.neat.legacy.graph.models import Triple
 from cognite.neat.legacy.graph.stores._rdf_to_graph import rdf_file_to_graph
 from cognite.neat.legacy.rules.models.rules import Rules
@@ -25,7 +25,11 @@ prom_qsm = Summary("store_query_time_summary_legacy", "Time spent processing que
 prom_sq = Gauge("store_single_query_time_legacy", "Time spent processing a single query", ["query"])
 
 MIMETypes: TypeAlias = Literal[
-    "application/rdf+xml", "text/turtle", "application/n-triple", "application/n-quads", "application/trig"
+    "application/rdf+xml",
+    "text/turtle",
+    "application/n-triple",
+    "application/n-quads",
+    "application/trig",
 ]
 
 
@@ -35,7 +39,8 @@ class NeatGraphStoreBase(ABC):
 
     Args:
         graph : Instance of rdflib.Graph class for graph storage
-        base_prefix : Used as a base prefix for graph namespace, allowing querying graph data using a shortform of a URI
+        base_prefix : Used as a base prefix for graph namespace,
+                      allowing querying graph data using a short form of a URI
         namespace : Namespace (aka URI) used to resolve any relative URI in the graph
         prefixes : Dictionary of additional prefixes used and bounded to the graph
     """
@@ -47,12 +52,12 @@ class NeatGraphStoreBase(ABC):
         graph: Graph | None = None,
         base_prefix: str = "",  # usually empty
         namespace: Namespace = DEFAULT_NAMESPACE,
-        prefixes: dict = PREFIXES,
+        prefixes: dict[str, Namespace] | None = None,
     ):
         self.graph = graph or Graph()
         self.base_prefix: str = base_prefix
         self.namespace: Namespace = namespace
-        self.prefixes: dict[str, Namespace] = prefixes
+        self.prefixes: dict[str, Namespace] = prefixes or get_default_prefixes()
 
         self.rdf_store_query_url: str | None = None
         self.rdf_store_update_url: str | None = None
@@ -164,7 +169,10 @@ class NeatGraphStoreBase(ABC):
         return None
 
     def import_from_file(
-        self, graph_file: Path, mime_type: MIMETypes = "application/rdf+xml", add_base_iri: bool = True
+        self,
+        graph_file: Path,
+        mime_type: MIMETypes = "application/rdf+xml",
+        add_base_iri: bool = True,
     ) -> None:
         """Imports graph data from file.
 
@@ -175,7 +183,10 @@ class NeatGraphStoreBase(ABC):
         """
         if add_base_iri:
             self.graph = rdf_file_to_graph(
-                self.graph, graph_file, base_namespace=self.namespace, prefixes=self.prefixes
+                self.graph,
+                graph_file,
+                base_namespace=self.namespace,
+                prefixes=self.prefixes,
             )
         else:
             self.graph = rdf_file_to_graph(self.graph, graph_file, prefixes=self.prefixes)
@@ -290,7 +301,12 @@ class NeatGraphStoreBase(ABC):
             "rdf_store_update_url": self.rdf_store_update_url,
         }
 
-    def add_triples(self, triples: list[Triple] | set[Triple], batch_size: int = 10_000, verbose: bool = False):
+    def add_triples(
+        self,
+        triples: list[Triple] | set[Triple],
+        batch_size: int = 10_000,
+        verbose: bool = False,
+    ):
         """Adds triples to the graph store in batches.
 
         Args:

cognite/neat/legacy/graph/stores/_graphdb_store.py

@@ -4,7 +4,7 @@ import requests
 from rdflib import Graph, Namespace
 from rdflib.plugins.stores.sparqlstore import SPARQLUpdateStore
 
-from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
+from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
 
 from ._base import NeatGraphStoreBase
 
@@ -28,8 +28,9 @@ class GraphDBStore(NeatGraphStoreBase):
         graph: Graph | None = None,
         base_prefix: str = "",  # usually empty
         namespace: Namespace = DEFAULT_NAMESPACE,
-        prefixes: dict = PREFIXES,
+        prefixes: dict[str, Namespace] | None = None,
     ):
+        prefixes = prefixes if prefixes else get_default_prefixes()
         super().__init__(graph, base_prefix, namespace, prefixes)
         self.graph_db_rest_url: str = "http://localhost:7200"
 

cognite/neat/legacy/graph/stores/_memory_store.py

@@ -2,7 +2,7 @@ import logging
 
 from rdflib import Graph, Namespace
 
-from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
+from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
 
 from ._base import NeatGraphStoreBase
 
@@ -26,9 +26,9 @@ class MemoryStore(NeatGraphStoreBase):
         graph: Graph | None = None,
         base_prefix: str = "",  # usually empty
         namespace: Namespace = DEFAULT_NAMESPACE,
-        prefixes: dict = PREFIXES,
+        prefixes: dict[str, Namespace] | None = None,
     ):
-        # Init repeated to get nice docstring
+        prefixes = prefixes if prefixes else get_default_prefixes()  # Init repeated to get nice docstring
         super().__init__(graph, base_prefix, namespace, prefixes)
 
     def _set_graph(self):

cognite/neat/legacy/graph/stores/_oxigraph_store.py

@@ -5,7 +5,7 @@ from pathlib import Path
 
 from rdflib import Graph, Namespace
 
-from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
+from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
 from cognite.neat.utils.auxiliary import local_import
 
 from ._base import MIMETypes, NeatGraphStoreBase
@@ -29,8 +29,9 @@ class OxiGraphStore(NeatGraphStoreBase):
         graph: Graph | None = None,
         base_prefix: str = "",  # usually empty
         namespace: Namespace = DEFAULT_NAMESPACE,
-        prefixes: dict = PREFIXES,
+        prefixes: dict[str, Namespace] | None = None,
     ):
+        prefixes = prefixes if prefixes else get_default_prefixes()
         super().__init__(graph, base_prefix, namespace, prefixes)
 
     def _set_graph(self) -> None:
@@ -44,7 +45,7 @@ class OxiGraphStore(NeatGraphStoreBase):
         for i in range(4):
             try:
                 oxstore = pyoxigraph.Store(
-                    path=str(self.internal_storage_dir) if self.internal_storage_dir else None
+                    path=(str(self.internal_storage_dir) if self.internal_storage_dir else None)
                 )  # Store (Rust object) accepts only str as path and not Path.
                 break
             except OSError as e:
@@ -76,7 +77,10 @@ class OxiGraphStore(NeatGraphStoreBase):
         logging.info("GraphStore restarted")
 
     def import_from_file(
-        self, graph_file: Path, mime_type: MIMETypes = "application/rdf+xml", add_base_iri: bool = True
+        self,
+        graph_file: Path,
+        mime_type: MIMETypes = "application/rdf+xml",
+        add_base_iri: bool = True,
     ) -> None:
         """Imports graph data from file.
 

cognite/neat/legacy/graph/stores/_rdf_to_graph.py

@@ -2,7 +2,7 @@ from pathlib import Path
 
 from rdflib import Graph, Namespace
 
-from cognite.neat.constants import PREFIXES
+from cognite.neat.constants import get_default_prefixes
 
 
 def rdf_file_to_graph(
@@ -10,7 +10,7 @@ def rdf_file_to_graph(
     filepath: Path,
     base_prefix: str | None = None,
     base_namespace: Namespace | None = None,
-    prefixes: dict[str, Namespace] = PREFIXES,
+    prefixes: dict[str, Namespace] | None = None,
 ) -> Graph:
     """Created rdflib Graph instance loaded with RDF triples from file
 
@@ -18,13 +18,15 @@ def rdf_file_to_graph(
         filepath: Path to the RDF file
         base_prefix: base prefix for URIs. Defaults to None.
         base_namespace: base namespace for URIs . Defaults to None.
-        prefixes: Dictionary of prefixes to bind to graph. Defaults to PREFIXES.
+        prefixes: Dictionary of prefixes to bind to graph. Defaults to internal set of prefixes.
         graph: Graph instance to load RDF triples into. Defaults to None.
 
     Returns:
         Graph instance loaded with RDF triples from file
     """
 
+    prefixes = prefixes if prefixes else get_default_prefixes()
+
     if filepath.is_file():
         graph.parse(filepath, publicID=base_namespace)
     else:

cognite/neat/legacy/graph/transformations/query_generator/sparql.py

@@ -6,7 +6,7 @@ from typing import cast
 from rdflib import Graph, Namespace
 from rdflib.term import URIRef
 
-from cognite.neat.constants import PREFIXES
+from cognite.neat.constants import get_default_prefixes
 from cognite.neat.legacy.rules.analysis import get_classes_with_properties
 from cognite.neat.legacy.rules.models._base import Triple
 from cognite.neat.legacy.rules.models.rdfpath import (
@@ -24,7 +24,7 @@ from cognite.neat.legacy.rules.models.rules import Rules
 from cognite.neat.utils.utils import remove_namespace_from_uri
 
 
-def _generate_prefix_header(prefixes: dict[str, Namespace] = PREFIXES) -> str:
+def _generate_prefix_header(prefixes: dict[str, Namespace] | None = None) -> str:
     """Generate prefix header which is added to SPARQL query and allows for shorten query statements
 
     Parameters
@@ -37,11 +37,15 @@ def _generate_prefix_header(prefixes: dict[str, Namespace] = PREFIXES) -> str:
     str
         Prefix header
     """
+    prefixes = prefixes if prefixes else get_default_prefixes()
     return "".join(f"PREFIX {key}:<{value}>\n" for key, value in prefixes.items())
 
 
 def _get_predicate_id(
-    graph: Graph, subject_type_id: str, object_type_id: str, prefixes: dict[str, Namespace] = PREFIXES
+    graph: Graph,
+    subject_type_id: str,
+    object_type_id: str,
+    prefixes: dict[str, Namespace] | None = None,
 ) -> URIRef:
     """Returns predicate (aka property) URI (i.e., ID) that connects subject and object
 
@@ -61,6 +65,7 @@ def _get_predicate_id(
     URIRef
         ID of predicate (aka property) connecting subject and object
     """
+    prefixes = prefixes if prefixes else get_default_prefixes()
     query = """
 
         SELECT ?predicateTypeID
@@ -122,11 +127,21 @@ def _get_hop_triples(graph, path: Hop, prefixes) -> list[Triple]:
     if previous_step.property:
         triples.extend(
             [
-                Triple(subject=f"?{previous_step.class_.suffix}ID", predicate="a", object=previous_step.class_.id),
                 Triple(
-                    subject=f"?{previous_step.class_.suffix}ID", predicate=previous_step.property.id, object="?object"
+                    subject=f"?{previous_step.class_.suffix}ID",
+                    predicate="a",
+                    object=previous_step.class_.id,
+                ),
+                Triple(
+                    subject=f"?{previous_step.class_.suffix}ID",
+                    predicate=previous_step.property.id,
+                    object="?object",
+                ),
+                Triple(
+                    subject="?predicate",
+                    predicate="a",
+                    object=previous_step.property.id,
                 ),
-                Triple(subject="?predicate", predicate="a", object=previous_step.property.id),
             ]
         )
     else:
@@ -139,7 +154,7 @@ def _get_hop_triples(graph, path: Hop, prefixes) -> list[Triple]:
     return triples
 
 
-def _get_path_triples(graph: Graph, traversal: Traversal, prefixes: dict[str, Namespace] = PREFIXES) -> list[Triple]:
+def _get_path_triples(graph: Graph, traversal: Traversal, prefixes: dict[str, Namespace] | None = None) -> list[Triple]:
     """Creates triples subject-predicate-object from declarative graph traversal path
 
     Parameters
@@ -227,7 +242,9 @@ def _generate_all_references_query_statement(
 
 
 def _generate_single_property_query_statement(
-    subject: str, predicate: str, query_template: str = SINGLE_PROPERTY_SPARQL_QUERY_TEMPLATE
+    subject: str,
+    predicate: str,
+    query_template: str = SINGLE_PROPERTY_SPARQL_QUERY_TEMPLATE,
 ) -> str:
     query_insertions = "\n".join([f"\t\t?subject a {subject} .", f"\t\t?subject {predicate} ?object ."])
 
@@ -257,7 +274,7 @@ def _generate_hop_query_statement(triples: list[Triple], query_template: str = B
 def build_sparql_query(
     graph: Graph,
     traversal_path: str | Traversal,
-    prefixes: dict[str, Namespace] = PREFIXES,
+    prefixes: dict[str, Namespace] | None = None,
     insert_prefixes: bool = False,
 ) -> str:
     """Builds SPARQL query based on declarative traversal path
@@ -276,7 +293,7 @@ def build_sparql_query(
     str
         SPARQL query
     """
-
+    prefixes = prefixes if prefixes else get_default_prefixes()
     traversal = parse_traversal(traversal_path) if isinstance(traversal_path, str) else traversal_path
     triples = _get_path_triples(graph, traversal, prefixes)
 
@@ -297,7 +314,10 @@ def build_sparql_query(
     for prefix, URI in prefixes.items():
         query = query.replace(URI, f"{prefix}:")
 
-    return query.replace("insertPrefixes\n\n", _generate_prefix_header(prefixes) if insert_prefixes else "")
+    return query.replace(
+        "insertPrefixes\n\n",
+        _generate_prefix_header(prefixes) if insert_prefixes else "",
+    )
 
 
 def compress_uri(uri: URIRef, prefixes: dict) -> str:
@@ -420,7 +440,8 @@ def _add_filter(class_instances, query_template):
     if class_instances:
         class_instances_formatted = [f"<{instance}>" for instance in class_instances]
         query_template = query_template.replace(
-            "insert_filter", f"\n\nFILTER (?subject IN ({', '.join(class_instances_formatted)}))"
+            "insert_filter",
+            f"\n\nFILTER (?subject IN ({', '.join(class_instances_formatted)}))",
         )
     else:
         query_template = query_template.replace("insert_filter", "")
@@ -439,7 +460,10 @@ def _triples2sparql_statement(triples: list[Triple]):
 
 
 def _to_construct_triples(
-    graph: Graph, class_: str, transformation_rules: Rules, properties_optional: bool = True
+    graph: Graph,
+    class_: str,
+    transformation_rules: Rules,
+    properties_optional: bool = True,
 ) -> tuple[list[Triple], list[Triple]]:
     """Converts class definition to CONSTRUCT triples which are used to generate CONSTRUCT query
 
@@ -485,7 +509,10 @@ def _to_construct_triples(
         # by binding them to certain property
         if isinstance(traversal, AllReferences):
             graph_pattern_triple = Triple(
-                subject="BIND(?subject", predicate="AS", object=f"{graph_template_triple.object})", optional=False
+                subject="BIND(?subject",
+                predicate="AS",
+                object=f"{graph_template_triple.object})",
+                optional=False,
             )
 
         elif isinstance(traversal, SingleProperty):
@@ -510,7 +537,13 @@ def _to_construct_triples(
 
     # add first triple for graph pattern stating type of object
     patterns.insert(
-        0, Triple(subject="?subject", predicate="a", object=_most_occurring_element(class_ids), optional=False)
+        0,
+        Triple(
+            subject="?subject",
+            predicate="a",
+            object=_most_occurring_element(class_ids),
+            optional=False,
+        ),
     )
 
     return templates, patterns

cognite/neat/legacy/rules/importers/_graph2rules.py

@@ -10,11 +10,15 @@ from typing import cast
 import pandas as pd
 from rdflib import Graph, Literal, Namespace, URIRef
 
-from cognite.neat.constants import PREFIXES
+from cognite.neat.constants import get_default_prefixes
 from cognite.neat.legacy.rules import exceptions
 from cognite.neat.legacy.rules.exporters._rules2rules import to_dms_name
 from cognite.neat.legacy.rules.models.tables import Tables
-from cognite.neat.utils.utils import get_namespace, remove_namespace_from_uri, uri_to_short_form
+from cognite.neat.utils.utils import (
+    get_namespace,
+    remove_namespace_from_uri,
+    uri_to_short_form,
+)
 
 from ._base import BaseImporter
 
@@ -74,11 +78,26 @@ def _create_default_properties_parsing_config() -> dict[str, tuple[str, ...]]:
 
 def _create_default_classes_parsing_config() -> dict[str, tuple[str, ...]]:
     # TODO: these are to be read from Class pydantic model
-    return {"header": ("Class", "Description", "Parent Class", "Source", "Source Entity Name", "Match Type", "Comment")}
+    return {
+        "header": (
+            "Class",
+            "Description",
+            "Parent Class",
+            "Source",
+            "Source Entity Name",
+            "Match Type",
+            "Comment",
+        )
+    }
 
 
 def _parse_prefixes_df(prefixes: dict[str, Namespace]) -> pd.DataFrame:
-    return pd.DataFrame.from_dict({"Prefix": list(prefixes.keys()), "URI": [str(uri) for uri in prefixes.values()]})
+    return pd.DataFrame.from_dict(
+        {
+            "Prefix": list(prefixes.keys()),
+            "URI": [str(uri) for uri in prefixes.values()],
+        }
+    )
 
 
 def _parse_metadata_df() -> pd.DataFrame:
@@ -169,7 +188,7 @@ def _graph_to_data_model_dict(graph: Graph, max_number_of_instance: int = -1) ->
     """
     data_model: dict[str, dict] = {}
 
-    prefixes: dict[str, Namespace] = PREFIXES
+    prefixes: dict[str, Namespace] = get_default_prefixes()
 
     for class_ in _get_class_ids(graph):
         _add_uri_namespace_to_prefixes(class_, prefixes)
@@ -183,10 +202,18 @@ def _graph_to_data_model_dict(graph: Graph, max_number_of_instance: int = -1) ->
             )
             class_name = f"{class_name}_{len(data_model)+1}"
 
-        data_model[class_name] = {"properties": {}, "uri": uri_to_short_form(class_, prefixes)}
+        data_model[class_name] = {
+            "properties": {},
+            "uri": uri_to_short_form(class_, prefixes),
+        }
 
         for instance in _get_class_instance_ids(graph, class_, max_number_of_instance):
-            for property_, occurrence, data_type, object_type in _define_instance_properties(graph, instance):
+            for (
+                property_,
+                occurrence,
+                data_type,
+                object_type,
+            ) in _define_instance_properties(graph, instance):
                 property_name = remove_namespace_from_uri(property_)
                 _add_uri_namespace_to_prefixes(property_, prefixes)
 

cognite/neat/legacy/rules/models/raw_rules.py

@@ -9,11 +9,17 @@ from pydantic import field_validator
 from pydantic_core import ErrorDetails, ValidationError
 from rdflib import Namespace
 
-from cognite.neat.constants import PREFIXES
+from cognite.neat.constants import get_default_prefixes
 from cognite.neat.exceptions import wrangle_warnings
 
 # rules model and model components:
-from cognite.neat.legacy.rules.models.rules import Class, Metadata, Property, RuleModel, Rules
+from cognite.neat.legacy.rules.models.rules import (
+    Class,
+    Metadata,
+    Property,
+    RuleModel,
+    Rules,
+)
 from cognite.neat.legacy.rules.models.tables import Tables
 
 # importers:
@@ -36,7 +42,7 @@ class RawRules(RuleModel):
         classes: Classes defined in the data model
         properties: Class properties defined in the data model with accompanying transformation rules
                     to transform data from source to target representation
-        prefixes: Prefixes used in the data model. Defaults to PREFIXES
+        prefixes: Prefixes used in the data model. Defaults to internal set of prefixes
         instances: Instances defined in the data model. Defaults to None
     """
 
@@ -242,7 +248,9 @@ def _raw_tables_to_rules_dict(raw_tables: RawRules, validators_to_skip: set | No
         "metadata": _metadata_table2dict(raw_tables.Metadata),
         "classes": _classes_table2dict(raw_tables.Classes),
         "properties": _properties_table2dict(raw_tables.Properties),
-        "prefixes": PREFIXES if raw_tables.Prefixes.empty else _prefixes_table2dict(raw_tables.Prefixes),
+        "prefixes": (
+            get_default_prefixes() if raw_tables.Prefixes.empty else _prefixes_table2dict(raw_tables.Prefixes)
+        ),
     }
 
     rules_dict["instances"] = (
@@ -272,11 +280,15 @@ def _metadata_table2dict(meta_df: pd.DataFrame) -> dict[str, Any]:
     return metadata_dict
 
 
-def _classes_table2dict(classes_df: pd.DataFrame) -> dict[Any | None, dict[Hashable, Any]]:
+def _classes_table2dict(
+    classes_df: pd.DataFrame,
+) -> dict[Any | None, dict[Hashable, Any]]:
     return {class_.get("Class"): class_ for class_ in classes_df.to_dict(orient="records")}
 
 
-def _properties_table2dict(properties_df: pd.DataFrame) -> dict[str, dict[Hashable, Any]]:
+def _properties_table2dict(
+    properties_df: pd.DataFrame,
+) -> dict[str, dict[Hashable, Any]]:
     return {f"row {i+3}": property_ for i, property_ in enumerate(properties_df.to_dict(orient="records"))}