cognite-neat 0.74.0__tar.gz → 0.75.1__tar.gz

{cognite_neat-0.74.0 → cognite_neat-0.75.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.74.0
+Version: 0.75.1
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.75.1/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.75.1"

{cognite_neat-0.74.0 → cognite_neat-0.75.1}/cognite/neat/app/api/routers/core.py

@@ -9,8 +9,7 @@ from fastapi import APIRouter, UploadFile
 
 from cognite.neat.app.api.configuration import NEAT_APP
 from cognite.neat.rules import exporters, importers
-from cognite.neat.rules.models._rules import DMSRules
-from cognite.neat.rules.models._rules.base import RoleTypes
+from cognite.neat.rules.models.rules import DMSRules, RoleTypes
 
 router = APIRouter()
 

{cognite_neat-0.74.0 → cognite_neat-0.75.1}/cognite/neat/app/api/routers/data_exploration.py

@@ -16,7 +16,7 @@ from cognite.neat.app.api.data_classes.rest import (
 )
 from cognite.neat.app.api.utils.data_mapping import rdf_result_to_api_response
 from cognite.neat.app.api.utils.query_templates import query_templates
-from cognite.neat.graph.transformation import query_generator
+from cognite.neat.legacy.graph.transformations import query_generator
 from cognite.neat.utils.utils import remove_namespace
 from cognite.neat.workflows.steps.data_contracts import RulesData, SolutionGraph, SourceGraph
 

{cognite_neat-0.74.0 → cognite_neat-0.75.1}/cognite/neat/app/api/routers/rules.py

@@ -7,10 +7,12 @@ from rdflib import Namespace
 
 from cognite.neat.app.api.configuration import NEAT_APP
 from cognite.neat.app.api.data_classes.rest import TransformationRulesUpdateRequest
-from cognite.neat.rules import exporter, importer, importers
-from cognite.neat.rules.models._base import EntityTypes
-from cognite.neat.rules.models._rules.base import RoleTypes
-from cognite.neat.rules.models.rules import Class, Classes, Metadata, Properties, Property, Rules
+from cognite.neat.legacy.rules import exporters as legacy_exporters
+from cognite.neat.legacy.rules import importers as legacy_importers
+from cognite.neat.legacy.rules.models._base import EntityTypes
+from cognite.neat.legacy.rules.models.rules import Class, Classes, Metadata, Properties, Property, Rules
+from cognite.neat.rules import importers
+from cognite.neat.rules.models.rules import RoleTypes
 from cognite.neat.workflows.steps.data_contracts import RulesData
 from cognite.neat.workflows.steps.lib.rules_exporter import RulesToExcel
 from cognite.neat.workflows.steps.lib.rules_importer import ExcelToRules
@@ -90,7 +92,9 @@ def get_rules(
     try:
         # Trying to load rules V1
         if rules_schema_version == "" or rules_schema_version == "v1":
-            rules = cast(Rules, importer.ExcelImporter(path).to_rules(return_report=False, skip_validation=False))
+            rules = cast(
+                Rules, legacy_importers.ExcelImporter(path).to_rules(return_report=False, skip_validation=False)
+            )
             properties = [
                 {
                     "class": value.class_id,
@@ -148,7 +152,9 @@ def get_rules(
 def get_original_rules_from_file(file_name: str):
     # """Endpoint for retrieving raw transformation from file"""
     path = Path(NEAT_APP.config.rules_store_path) / file_name
-    rules = cast(Rules, importer.ExcelImporter(filepath=path).to_rules(return_report=False, skip_validation=False))
+    rules = cast(
+        Rules, legacy_importers.ExcelImporter(filepath=path).to_rules(return_report=False, skip_validation=False)
+    )
     return Response(content=rules.model_dump_json(), media_type="application/json")
 
 
@@ -193,5 +199,5 @@ def upsert_rules(request: TransformationRulesUpdateRequest):
         else:
             path = Path(NEAT_APP.config.data_store_path) / rules_file
 
-        exporter.ExcelExporter(rules=rules).export_to_file(path)
+        legacy_exporters.ExcelExporter(rules=rules).export_to_file(path)
     return {"status": "ok"}

{cognite_neat-0.74.0 → cognite_neat-0.75.1}/cognite/neat/constants.py

@@ -7,8 +7,8 @@ from cognite import neat
 PACKAGE_DIRECTORY = Path(neat.__file__).parent
 
 
-EXAMPLE_RULES = PACKAGE_DIRECTORY / "rules" / "examples"
-EXAMPLE_GRAPHS = PACKAGE_DIRECTORY / "graph" / "examples"
+EXAMPLE_RULES = PACKAGE_DIRECTORY / "legacy" / "rules" / "examples"
+EXAMPLE_GRAPHS = PACKAGE_DIRECTORY / "legacy" / "graph" / "examples"
 EXAMPLE_WORKFLOWS = PACKAGE_DIRECTORY / "workflows" / "examples"
 
 DEFAULT_NAMESPACE = Namespace("http://purl.org/cognite/neat#")

{cognite_neat-0.74.0 → cognite_neat-0.75.1}/cognite/neat/graph/extractors/_mock_graph_generator.py

@@ -13,10 +13,10 @@ import pandas as pd
 from rdflib import RDF, Literal, Namespace, URIRef
 
 from cognite.neat.graph.models import Triple
-from cognite.neat.rules._analysis._information_rules import InformationArchitectRulesAnalysis
-from cognite.neat.rules.models._rules import DMSRules, InformationRules
-from cognite.neat.rules.models._rules._types import ClassEntity, EntityTypes, XSDValueType
-from cognite.neat.rules.models._rules.information_rules import InformationProperty
+from cognite.neat.rules.analysis import InformationArchitectRulesAnalysis
+from cognite.neat.rules.models.rules import DMSRules, InformationRules
+from cognite.neat.rules.models.rules._information_rules import InformationProperty
+from cognite.neat.rules.models.rules._types import ClassEntity, EntityTypes, XSDValueType
 from cognite.neat.utils.utils import remove_namespace
 
 from ._base import BaseExtractor

cognite_neat-0.75.1/cognite/neat/graph/stores/_base.py

@@ -0,0 +1,363 @@
+import logging
+import sys
+import time
+from abc import ABC, abstractmethod
+from collections.abc import Iterable, Iterator
+from pathlib import Path
+from typing import Literal, TypeAlias, cast
+
+import pandas as pd
+from prometheus_client import Gauge, Summary
+from rdflib import Graph, Namespace, URIRef
+from rdflib.query import Result, ResultRow
+
+from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
+from cognite.neat.graph.models import Triple
+from cognite.neat.graph.stores._rdf_to_graph import rdf_file_to_graph
+
+if sys.version_info >= (3, 11):
+    pass
+else:
+    pass
+
+prom_qsm = Summary("store_query_time_summary", "Time spent processing queries", ["query"])
+prom_sq = Gauge("store_single_query_time", "Time spent processing a single query", ["query"])
+
+MIMETypes: TypeAlias = Literal[
+    "application/rdf+xml", "text/turtle", "application/n-triple", "application/n-quads", "application/trig"
+]
+
+
+class NeatGraphStoreBase(ABC):
+    """NeatGraphStore is a class that stores the graph and provides methods to read/write data it contains
+
+
+    Args:
+        graph : Instance of rdflib.Graph class for graph storage
+        base_prefix : Used as a base prefix for graph namespace, allowing querying graph data using a shortform of a URI
+        namespace : Namespace (aka URI) used to resolve any relative URI in the graph
+        prefixes : Dictionary of additional prefixes used and bounded to the graph
+    """
+
+    rdf_store_type: str
+
+    def __init__(
+        self,
+        graph: Graph | None = None,
+        base_prefix: str = "",  # usually empty
+        namespace: Namespace = DEFAULT_NAMESPACE,
+        prefixes: dict = PREFIXES,
+    ):
+        self.graph = graph or Graph()
+        self.base_prefix: str = base_prefix
+        self.namespace: Namespace = namespace
+        self.prefixes: dict[str, Namespace] = prefixes
+
+        self.rdf_store_query_url: str | None = None
+        self.rdf_store_update_url: str | None = None
+        self.returnFormat: str | None = None
+        self.df_cache: pd.DataFrame | None = None
+        self.internal_storage_dir: Path | None = None
+        self.graph_name: str | None = None
+        self.internal_storage_dir_orig: Path | None = None
+        self.storage_dirs_to_delete: list[Path] = []
+        self.queries = _Queries(self)
+
+    @abstractmethod
+    def _set_graph(self) -> None:
+        raise NotImplementedError()
+
+    def init_graph(
+        self,
+        rdf_store_query_url: str | None = None,
+        rdf_store_update_url: str | None = None,
+        graph_name: str | None = None,
+        base_prefix: str | None = None,
+        returnFormat: str = "csv",
+        internal_storage_dir: Path | None = None,
+    ):
+        """Initializes the graph.
+
+        Args:
+            rdf_store_query_url : URL towards which SPARQL query is executed, by default None
+            rdf_store_update_url : URL towards which SPARQL update is executed, by default None
+            graph_name : Name of graph, by default None
+            base_prefix : Base prefix for graph namespace to change if needed, by default None
+            returnFormat : Transport format of graph data between, by default "csv"
+            internal_storage_dir : Path to directory where internal storage is located,
+                                   by default None (in-memory storage).
+
+        !!! note "internal_storage_dir"
+            Used only for Oxigraph
+        """
+        logging.info("Initializing NeatGraphStore")
+        self.rdf_store_query_url = rdf_store_query_url
+        self.rdf_store_update_url = rdf_store_update_url
+        self.graph_name = graph_name
+        self.returnFormat = returnFormat
+        self.internal_storage_dir = Path(internal_storage_dir) if internal_storage_dir else None
+        self.internal_storage_dir_orig = (
+            self.internal_storage_dir if self.internal_storage_dir_orig is None else self.internal_storage_dir_orig
+        )
+
+        self._set_graph()
+
+        if self.prefixes:
+            for prefix, namespace in self.prefixes.items():
+                logging.info("Adding prefix %s with namespace %s", prefix, namespace)
+                self.graph.bind(prefix, namespace)
+
+        if base_prefix:
+            self.base_prefix = base_prefix
+
+        self.graph.bind(self.base_prefix, self.namespace)
+        logging.info("Adding prefix %s with namespace %s", self.base_prefix, self.namespace)
+        logging.info("Graph initialized")
+
+    def reinitialize_graph(self):
+        """Reinitialize the graph."""
+        self.init_graph(
+            self.rdf_store_query_url,
+            self.rdf_store_update_url,
+            self.graph_name,
+            self.base_prefix,
+            self.returnFormat,
+            self.internal_storage_dir,
+        )
+
+    def upsert_prefixes(self, prefixes: dict[str, Namespace]) -> None:
+        """Adds prefixes to the graph store."""
+        self.prefixes.update(prefixes)
+        for prefix, namespace in prefixes.items():
+            logging.info("Adding prefix %s with namespace %s", prefix, namespace)
+            self.graph.bind(prefix, namespace)
+
+    def close(self) -> None:
+        """Closes the graph."""
+        # Can be overridden in subclasses
+        return None
+
+    def restart(self) -> None:
+        """Restarts the graph"""
+        # Can be overridden in subclasses
+        return None
+
+    def import_from_file(
+        self, graph_file: Path, mime_type: MIMETypes = "application/rdf+xml", add_base_iri: bool = True
+    ) -> None:
+        """Imports graph data from file.
+
+        Args:
+            graph_file : File path to file containing graph data, by default None
+            mime_type : MIME type of graph data, by default "application/rdf+xml"
+            add_base_iri : Add base IRI to graph, by default True
+        """
+        if add_base_iri:
+            self.graph = rdf_file_to_graph(
+                self.graph, graph_file, base_namespace=self.namespace, prefixes=self.prefixes
+            )
+        else:
+            self.graph = rdf_file_to_graph(self.graph, graph_file, prefixes=self.prefixes)
+        return None
+
+    def get_graph(self) -> Graph:
+        """Returns the graph."""
+        return self.graph
+
+    def set_graph(self, graph: Graph):
+        """Sets the graph."""
+        self.graph = graph
+
+    def query(self, query: str) -> Result:
+        """Returns the result of the query."""
+        start_time = time.perf_counter()
+        result = self.graph.query(query)
+        stop_time = time.perf_counter()
+        elapsed_time = stop_time - start_time
+        prom_qsm.labels("query").observe(elapsed_time)
+        prom_sq.labels("query").set(elapsed_time)
+        return result
+
+    def serialize(self, *args, **kwargs):
+        """Serializes the graph."""
+        return self.graph.serialize(*args, **kwargs)
+
+    def query_delayed(self, query) -> Iterable[Triple]:
+        """Returns the result of the query, but does not execute it immediately.
+
+        The query is not executed until the result is iterated over.
+
+        Args:
+            query: SPARQL query to execute
+
+        Returns:
+            An iterable of triples
+
+        """
+        return _DelayedQuery(self.graph, query)
+
+    @abstractmethod
+    def drop(self) -> None:
+        """Drops the graph."""
+        raise NotImplementedError()
+
+    def garbage_collector(self) -> None:
+        """Garbage collection of the graph store."""
+        # Can be overridden in subclasses
+        return None
+
+    def query_to_dataframe(
+        self,
+        query: str,
+        column_mapping: dict | None = None,
+        save_to_cache: bool = False,
+        index_column: str = "instance",
+    ) -> pd.DataFrame:
+        """Returns the result of the query as a dataframe.
+
+        Args:
+            query: SPARQL query to execute
+            column_mapping: Columns name mapping, by default None
+            save_to_cache: Save result of query to cache, by default False
+            index_column: Indexing column , by default "instance"
+
+        Returns:
+            Dataframe with result of query
+        """
+
+        if column_mapping is None:
+            column_mapping = {0: "instance", 1: "property", 2: "value"}
+
+        result = self.graph.query(query, DEBUG=False)
+        df_cache = pd.DataFrame(list(result))
+        df_cache.rename(columns=column_mapping, inplace=True)
+        df_cache[index_column] = df_cache[index_column].apply(lambda x: str(x))
+        if save_to_cache:
+            self.df_cache = df_cache
+        return df_cache
+
+    def commit(self):
+        """Commits the graph."""
+        self.graph.commit()
+
+    def get_df(self) -> pd.DataFrame:
+        """Returns the cached dataframe."""
+        if self.df_cache is None:
+            raise ValueError("Cache is empty. Run query_to_dataframe() first with save_to_cache.")
+        return self.df_cache
+
+    def get_instance_properties_from_cache(self, instance_id: str) -> pd.DataFrame:
+        """Returns the properties of an instance."""
+        if self.df_cache is None:
+            raise ValueError("Cache is empty. Run query_to_dataframe() first with save_to_cache.")
+        return self.df_cache.loc[self.df_cache["instance"] == instance_id]
+
+    def print_triples(self):
+        """Prints the triples of the graph."""
+        for subj, pred, obj in self.graph:
+            logging.info(f"Triple: {subj} {pred} {obj}")
+
+    def diagnostic_report(self):
+        """Returns the dictionary representation graph diagnostic data ."""
+        return {
+            "rdf_store_type": self.rdf_store_type,
+            "base_prefix": self.base_prefix,
+            "namespace": self.namespace,
+            "prefixes": self.prefixes,
+            "internal_storage_dir": self.internal_storage_dir,
+            "rdf_store_query_url": self.rdf_store_query_url,
+            "rdf_store_update_url": self.rdf_store_update_url,
+        }
+
+    def add_triples(self, triples: list[Triple] | set[Triple], batch_size: int = 10_000, verbose: bool = False):
+        """Adds triples to the graph store in batches.
+
+        Args:
+            triples: list of triples to be added to the graph store
+            batch_size: Batch size of triples per commit, by default 10_000
+            verbose: Verbose mode, by default False
+        """
+
+        commit_counter = 0
+        if verbose:
+            logging.info(f"Committing total of {len(triples)} triples to knowledge graph!")
+        total_number_of_triples = len(triples)
+        number_of_uploaded_triples = 0
+
+        def check_commit(force_commit: bool = False):
+            """Commit nodes to the graph if batch counter is reached or if force_commit is True"""
+            nonlocal commit_counter
+            nonlocal number_of_uploaded_triples
+            if force_commit:
+                number_of_uploaded_triples += commit_counter
+                self.graph.commit()
+                if verbose:
+                    logging.info(f"Committed {number_of_uploaded_triples} of {total_number_of_triples} triples")
+                return
+            commit_counter += 1
+            if commit_counter >= batch_size:
+                number_of_uploaded_triples += commit_counter
+                self.graph.commit()
+                if verbose:
+                    logging.info(f"Committed {number_of_uploaded_triples} of {total_number_of_triples} triples")
+                commit_counter = 0
+
+        for triple in triples:
+            self.graph.add(triple)
+            check_commit()
+
+        check_commit(force_commit=True)
+
+
+class _DelayedQuery(Iterable):
+    def __init__(self, graph_ref: Graph, query: str):
+        self.graph_ref = graph_ref
+        self.query = query
+
+    def __iter__(self) -> Iterator[Triple]:
+        start_time = time.perf_counter()
+        result = self.graph_ref.query(self.query)
+        stop_time = time.perf_counter()
+        elapsed_time = stop_time - start_time
+        prom_qsm.labels("query").observe(elapsed_time)
+        prom_sq.labels("query").set(elapsed_time)
+        return cast(Iterator[Triple], iter(result))
+
+
+class _Queries:
+    """Helper class for storing standard queries for the graph store."""
+
+    def __init__(self, store: NeatGraphStoreBase):
+        self.store = store
+
+    def list_instances_ids_of_class(self, class_uri: URIRef, limit: int = -1) -> list[URIRef]:
+        """Get instances ids for a given class
+
+        Args:
+            class_uri: Class for which instances are to be found
+            limit: Max number of instances to return, by default -1 meaning all instances
+
+        Returns:
+            List of class instance URIs
+        """
+        query_statement = "SELECT DISTINCT ?subject WHERE { ?subject a <class> .} LIMIT X".replace(
+            "class", class_uri
+        ).replace("LIMIT X", "" if limit == -1 else f"LIMIT {limit}")
+        return [cast(tuple, res)[0] for res in list(self.store.query(query_statement))]
+
+    def list_instances_of_type(self, class_uri: URIRef) -> list[ResultRow]:
+        """Get all triples for instances of a given class
+
+        Args:
+            class_uri: Class for which instances are to be found
+
+        Returns:
+            List of triples for instances of the given class
+        """
+        query = (
+            f"SELECT ?instance ?prop ?value "
+            f"WHERE {{ ?instance rdf:type <{class_uri}> . ?instance ?prop ?value . }} order by ?instance "
+        )
+        logging.info(query)
+        # Select queries gives an iterable of result rows
+        return cast(list[ResultRow], list(self.store.query(query)))

cognite_neat-0.75.1/cognite/neat/legacy/graph/__init__.py

@@ -0,0 +1,3 @@
+from .stores import NeatGraphStoreBase
+
+__all__ = ["NeatGraphStoreBase"]