cognite-neat 0.79.0__py3-none-any.whl → 0.80.0__py3-none-any.whl

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.79.0"
+__version__ = "0.80.0"

cognite/neat/graph/__init__.py

@@ -1,3 +1,3 @@
-from .stores import NeatGraphStoreBase
+from .stores import NeatGraphStore
 
-__all__ = ["NeatGraphStoreBase"]
+__all__ = ["NeatGraphStore"]

cognite/neat/graph/_shared.py

@@ -0,0 +1,5 @@
+from typing import Literal, TypeAlias
+
+MIMETypes: TypeAlias = Literal[
+    "application/rdf+xml", "text/turtle", "application/n-triple", "application/n-quads", "application/trig"
+]

cognite/neat/graph/extractors/__init__.py

@@ -6,6 +6,7 @@ from ._classic_cdf._relationships import RelationshipsExtractor
 from ._classic_cdf._sequences import SequencesExtractor
 from ._classic_cdf._timeseries import TimeSeriesExtractor
 from ._mock_graph_generator import MockGraphGenerator
+from ._rdf_file import RdfFileExtractor
 
 __all__ = [
     "AssetsExtractor",
@@ -16,4 +17,18 @@ __all__ = [
     "EventsExtractor",
     "FilesExtractor",
     "LabelsExtractor",
+    "RdfFileExtractor",
 ]
+
+
+TripleExtractors = (
+    AssetsExtractor
+    | MockGraphGenerator
+    | RelationshipsExtractor
+    | TimeSeriesExtractor
+    | SequencesExtractor
+    | EventsExtractor
+    | FilesExtractor
+    | LabelsExtractor
+    | RdfFileExtractor
+)

cognite/neat/graph/extractors/_rdf_file.py

@@ -0,0 +1,18 @@
+from pathlib import Path
+
+from rdflib import URIRef
+
+from cognite.neat.graph._shared import MIMETypes
+from cognite.neat.graph.extractors._base import BaseExtractor
+
+
+class RdfFileExtractor(BaseExtractor):
+    def __init__(
+        self,
+        filepath: Path,
+        mime_type: MIMETypes = "application/rdf+xml",
+        base_uri: URIRef | None = None,
+    ):
+        self.filepath = filepath
+        self.mime_type = mime_type
+        self.base_uri = base_uri

cognite/neat/graph/loaders/_base.py

@@ -6,7 +6,7 @@ from typing import ClassVar, Generic, Literal, TypeVar, overload
 from cognite.client import CogniteClient
 from cognite.client.data_classes.capabilities import Capability
 
-from cognite.neat.graph import NeatGraphStoreBase
+from cognite.neat.graph import NeatGraphStore
 from cognite.neat.graph.issues.loader import FailedAuthorizationError
 from cognite.neat.issues import NeatIssue, NeatIssueList
 from cognite.neat.utils.upload import UploadDiffsID, UploadResultIDs
@@ -18,7 +18,7 @@ class BaseLoader(ABC, Generic[T_Output]):
     _new_line = "\n"
     _encoding = "utf-8"
 
-    def __init__(self, graph_store: NeatGraphStoreBase):
+    def __init__(self, graph_store: NeatGraphStore):
         self.graph_store = graph_store
 
     @abstractmethod

cognite/neat/graph/loaders/_rdf2dms.py

@@ -17,7 +17,7 @@ from pydantic.main import Model
 
 from cognite.neat.graph._tracking import LogTracker, Tracker
 from cognite.neat.graph.issues import loader as loader_issues
-from cognite.neat.graph.stores import NeatGraphStoreBase
+from cognite.neat.graph.stores import NeatGraphStore
 from cognite.neat.issues import NeatIssue, NeatIssueList
 from cognite.neat.rules.models import DMSRules
 from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE
@@ -29,7 +29,7 @@ from ._base import CDFLoader
 class DMSLoader(CDFLoader[dm.InstanceApply]):
     def __init__(
         self,
-        graph_store: NeatGraphStoreBase,
+        graph_store: NeatGraphStore,
         data_model: dm.DataModel[dm.View] | None,
         instance_space: str,
         class_by_view_id: dict[ViewId, str] | None = None,
@@ -48,7 +48,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         cls,
         client: CogniteClient,
         data_model_id: dm.DataModelId,
-        graph_store: NeatGraphStoreBase,
+        graph_store: NeatGraphStore,
         instance_space: str,
     ) -> "DMSLoader":
         issues: list[NeatIssue] = []
@@ -61,7 +61,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         return cls(graph_store, data_model, instance_space, {}, issues)
 
     @classmethod
-    def from_rules(cls, rules: DMSRules, graph_store: NeatGraphStoreBase, instance_space: str) -> "DMSLoader":
+    def from_rules(cls, rules: DMSRules, graph_store: NeatGraphStore, instance_space: str) -> "DMSLoader":
         issues: list[NeatIssue] = []
         data_model: dm.DataModel[dm.View] | None = None
         try:

cognite/neat/graph/stores/__init__.py

@@ -1,13 +1,3 @@
-from ._base import NeatGraphStoreBase
-from ._graphdb_store import GraphDBStore
-from ._memory_store import MemoryStore
-from ._oxigraph_store import OxiGraphStore
+from ._base import NeatGraphStore
 
-STORE_BY_TYPE: dict[str, type[NeatGraphStoreBase]] = {}
-for store in NeatGraphStoreBase.__subclasses__():
-    STORE_BY_TYPE[store.rdf_store_type] = store  # type: ignore[type-abstract]
-
-del store  # Cleanup namespace
-AVAILABLE_STORES = set(STORE_BY_TYPE.keys())
-
-__all__ = ["NeatGraphStoreBase", "MemoryStore", "OxiGraphStore", "GraphDBStore", "STORE_BY_TYPE", "AVAILABLE_STORES"]
+__all__ = ["NeatGraphStore"]

cognite/neat/graph/stores/_base.py

@@ -1,291 +1,170 @@
-import logging
 import sys
-import time
-from abc import ABC, abstractmethod
-from collections.abc import Iterable, Iterator
+import warnings
+from collections.abc import Iterable
 from datetime import datetime
 from pathlib import Path
-from typing import Literal, TypeAlias, cast
+from typing import cast
 
-import pandas as pd
 import pytz
-from prometheus_client import Gauge, Summary
 from rdflib import RDF, Graph, Namespace, URIRef
-from rdflib.query import Result, ResultRow
+from rdflib.plugins.stores.sparqlstore import SPARQLUpdateStore
+from rdflib.query import ResultRow
 
-from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
+from cognite.neat.graph._shared import MIMETypes
+from cognite.neat.graph.extractors import RdfFileExtractor, TripleExtractors
 from cognite.neat.graph.models import Triple
-from cognite.neat.graph.stores._rdf_to_graph import rdf_file_to_graph
+from cognite.neat.rules.models.information import InformationRules
 from cognite.neat.utils import remove_namespace
+from cognite.neat.utils.auxiliary import local_import
 
 from ._provenance import Change, Provenance
 
-if sys.version_info >= (3, 11):
-    pass
+if sys.version_info < (3, 11):
+    from typing_extensions import Self
 else:
-    pass
+    from typing import Self
 
-prom_qsm = Summary("store_query_time_summary", "Time spent processing queries", ["query"])
-prom_sq = Gauge("store_single_query_time", "Time spent processing a single query", ["query"])
 
-MIMETypes: TypeAlias = Literal[
-    "application/rdf+xml", "text/turtle", "application/n-triple", "application/n-quads", "application/trig"
-]
-
-
-class NeatGraphStoreBase(ABC):
+class NeatGraphStore:
     """NeatGraphStore is a class that stores the graph and provides methods to read/write data it contains
 
 
     Args:
         graph : Instance of rdflib.Graph class for graph storage
-        base_prefix : Used as a base prefix for graph namespace, allowing querying graph data using a shortform of a URI
-        namespace : Namespace (aka URI) used to resolve any relative URI in the graph
-        prefixes : Dictionary of additional prefixes used and bounded to the graph
+        rules:
     """
 
     rdf_store_type: str
 
     def __init__(
         self,
-        graph: Graph | None = None,
-        base_prefix: str = "",  # usually empty
-        namespace: Namespace = DEFAULT_NAMESPACE,
-        prefixes: dict = PREFIXES,
+        graph: Graph,
+        rules: InformationRules | None = None,
     ):
         _start = datetime.now(pytz.utc)
-        self.graph = graph or Graph()
-        self.base_prefix: str = base_prefix
-        self.namespace: Namespace = namespace
-        self.prefixes: dict[str, Namespace] = prefixes
-
-        self.rdf_store_query_url: str | None = None
-        self.rdf_store_update_url: str | None = None
-        self.returnFormat: str | None = None
-        self.df_cache: pd.DataFrame | None = None
-        self.internal_storage_dir: Path | None = None
-        self.graph_name: str | None = None
-        self.internal_storage_dir_orig: Path | None = None
-        self.storage_dirs_to_delete: list[Path] = []
-        self.queries = _Queries(self)
+        self.graph = graph
         self.provenance = Provenance(
             [
                 Change.record(
                     activity=f"{type(self).__name__}.__init__",
                     start=_start,
                     end=datetime.now(pytz.utc),
-                    description="Initialize graph store",
+                    description=f"Initialize graph store as {type(self.graph.store).__name__}",
                 )
             ]
         )
+        self.rules = rules
 
-    @abstractmethod
-    def _set_graph(self) -> None:
-        raise NotImplementedError()
-
-    def init_graph(
-        self,
-        rdf_store_query_url: str | None = None,
-        rdf_store_update_url: str | None = None,
-        graph_name: str | None = None,
-        base_prefix: str | None = None,
-        returnFormat: str = "csv",
-        internal_storage_dir: Path | None = None,
-    ):
-        """Initializes the graph.
-
-        Args:
-            rdf_store_query_url : URL towards which SPARQL query is executed, by default None
-            rdf_store_update_url : URL towards which SPARQL update is executed, by default None
-            graph_name : Name of graph, by default None
-            base_prefix : Base prefix for graph namespace to change if needed, by default None
-            returnFormat : Transport format of graph data between, by default "csv"
-            internal_storage_dir : Path to directory where internal storage is located,
-                                   by default None (in-memory storage).
-
-        !!! note "internal_storage_dir"
-            Used only for Oxigraph
-        """
-        logging.info("Initializing NeatGraphStore")
-        self.rdf_store_query_url = rdf_store_query_url
-        self.rdf_store_update_url = rdf_store_update_url
-        self.graph_name = graph_name
-        self.returnFormat = returnFormat
-        self.internal_storage_dir = Path(internal_storage_dir) if internal_storage_dir else None
-        self.internal_storage_dir_orig = (
-            self.internal_storage_dir if self.internal_storage_dir_orig is None else self.internal_storage_dir_orig
-        )
+        if self.rules and self.rules.prefixes:
+            self._upsert_prefixes(self.rules.prefixes)
 
-        self._set_graph()
-
-        if self.prefixes:
-            for prefix, namespace in self.prefixes.items():
-                logging.info("Adding prefix %s with namespace %s", prefix, namespace)
-                self.graph.bind(prefix, namespace)
-
-        if base_prefix:
-            self.base_prefix = base_prefix
-        if self.base_prefix:
-            self.graph.bind(self.base_prefix, self.namespace)
-            logging.info("Adding prefix %s with namespace %s", self.base_prefix, self.namespace)
-        logging.info("Graph initialized")
-
-    def reinitialize_graph(self):
-        """Reinitialize the graph."""
-        self.init_graph(
-            self.rdf_store_query_url,
-            self.rdf_store_update_url,
-            self.graph_name,
-            self.base_prefix,
-            self.returnFormat,
-            self.internal_storage_dir,
-        )
+        self.queries = _Queries(self)
 
-    def upsert_prefixes(self, prefixes: dict[str, Namespace]) -> None:
+    def _upsert_prefixes(self, prefixes: dict[str, Namespace]) -> None:
         """Adds prefixes to the graph store."""
-        self.prefixes.update(prefixes)
+        _start = datetime.now(pytz.utc)
         for prefix, namespace in prefixes.items():
-            logging.info("Adding prefix %s with namespace %s", prefix, namespace)
             self.graph.bind(prefix, namespace)
 
-    def close(self) -> None:
-        """Closes the graph."""
-        # Can be overridden in subclasses
-        return None
-
-    def restart(self) -> None:
-        """Restarts the graph"""
-        # Can be overridden in subclasses
-        return None
-
-    def import_from_file(
-        self, graph_file: Path, mime_type: MIMETypes = "application/rdf+xml", add_base_iri: bool = True
-    ) -> None:
-        """Imports graph data from file.
-
-        Args:
-            graph_file : File path to file containing graph data, by default None
-            mime_type : MIME type of graph data, by default "application/rdf+xml"
-            add_base_iri : Add base IRI to graph, by default True
-        """
-        if add_base_iri:
-            self.graph = rdf_file_to_graph(
-                self.graph, graph_file, base_namespace=self.namespace, prefixes=self.prefixes
+        self.provenance.append(
+            Change.record(
+                activity=f"{type(self).__name__}._upsert_prefixes",
+                start=_start,
+                end=datetime.now(pytz.utc),
+                description="Upsert prefixes to graph store",
             )
-        else:
-            self.graph = rdf_file_to_graph(self.graph, graph_file, prefixes=self.prefixes)
-        return None
+        )
 
-    def get_graph(self) -> Graph:
-        """Returns the graph."""
-        return self.graph
+    @classmethod
+    def from_memory_store(cls, rules: InformationRules | None = None) -> "Self":
+        return cls(Graph(), rules)
 
-    def set_graph(self, graph: Graph):
-        """Sets the graph."""
-        self.graph = graph
+    @classmethod
+    def from_sparql_store(
+        cls,
+        query_endpoint: str | None = None,
+        update_endpoint: str | None = None,
+        returnFormat: str = "csv",
+        rules: InformationRules | None = None,
+    ) -> "Self":
+        store = SPARQLUpdateStore(
+            query_endpoint=query_endpoint,
+            update_endpoint=update_endpoint,
+            returnFormat=returnFormat,
+            context_aware=False,
+            postAsEncoded=False,
+            autocommit=False,
+        )
+        graph = Graph(store=store)
+        return cls(graph, rules)
+
+    @classmethod
+    def from_oxi_store(cls, storage_dir: Path | None = None, rules: InformationRules | None = None) -> "Self":
+        """Creates a NeatGraphStore from an Oxigraph store."""
+        local_import("pyoxigraph", "oxi")
+        import pyoxigraph
+
+        from cognite.neat.graph.stores._oxrdflib import OxigraphStore
+
+        # Adding support for both oxigraph in-memory and file-based storage
+        for i in range(4):
+            try:
+                oxi_store = pyoxigraph.Store(path=str(storage_dir) if storage_dir else None)
+                break
+            except OSError as e:
+                if "lock" in str(e) and i < 3:
+                    continue
+                raise e
+        else:
+            raise Exception("Error initializing Oxigraph store")
 
-    def query(self, query: str) -> Result:
-        """Returns the result of the query."""
-        start_time = time.perf_counter()
-        result = self.graph.query(query)
-        stop_time = time.perf_counter()
-        elapsed_time = stop_time - start_time
-        prom_qsm.labels("query").observe(elapsed_time)
-        prom_sq.labels("query").set(elapsed_time)
-        return result
+        graph = Graph(store=OxigraphStore(store=oxi_store))
+        graph.default_union = True
 
-    def serialize(self, *args, **kwargs):
-        """Serializes the graph."""
-        return self.graph.serialize(*args, **kwargs)
+        return cls(graph, rules)
 
-    def query_delayed(self, query) -> Iterable[Triple]:
-        """Returns the result of the query, but does not execute it immediately.
+    def write(self, extractor: TripleExtractors) -> None:
+        if isinstance(extractor, RdfFileExtractor):
+            self._parse_file(extractor.filepath, extractor.mime_type, extractor.base_uri)
+        else:
+            self._add_triples(extractor.extract())
 
-        The query is not executed until the result is iterated over.
+    def _parse_file(
+        self,
+        filepath: Path,
+        mime_type: MIMETypes = "application/rdf+xml",
+        base_uri: URIRef | None = None,
+    ) -> None:
+        """Imports graph data from file.
 
         Args:
-            query: SPARQL query to execute
-
-        Returns:
-            An iterable of triples
-
+            filepath : File path to file containing graph data, by default None
+            mime_type : MIME type of graph data, by default "application/rdf+xml"
+            add_base_iri : Add base IRI to graph, by default True
         """
-        return _DelayedQuery(self.graph, query)
 
-    @abstractmethod
-    def drop(self) -> None:
-        """Drops the graph."""
-        raise NotImplementedError()
+        # Oxigraph store, do not want to type hint this as it is an optional dependency
+        if type(self.graph.store).__name__ == "OxigraphStore":
 
-    def garbage_collector(self) -> None:
-        """Garbage collection of the graph store."""
-        # Can be overridden in subclasses
-        return None
+            def parse_to_oxi_store():
+                local_import("pyoxigraph", "oxi")
+                from cognite.neat.graph.stores._oxrdflib import OxigraphStore
 
-    def query_to_dataframe(
-        self,
-        query: str,
-        column_mapping: dict | None = None,
-        save_to_cache: bool = False,
-        index_column: str = "instance",
-    ) -> pd.DataFrame:
-        """Returns the result of the query as a dataframe.
-
-        Args:
-            query: SPARQL query to execute
-            column_mapping: Columns name mapping, by default None
-            save_to_cache: Save result of query to cache, by default False
-            index_column: Indexing column , by default "instance"
+                cast(OxigraphStore, self.graph.store)._inner.bulk_load(str(filepath), mime_type, base_iri=base_uri)  # type: ignore[attr-defined]
+                cast(OxigraphStore, self.graph.store)._inner.optimize()  # type: ignore[attr-defined]
 
-        Returns:
-            Dataframe with result of query
-        """
+            parse_to_oxi_store()
 
-        if column_mapping is None:
-            column_mapping = {0: "instance", 1: "property", 2: "value"}
-
-        result = self.graph.query(query, DEBUG=False)
-        df_cache = pd.DataFrame(list(result))
-        df_cache.rename(columns=column_mapping, inplace=True)
-        df_cache[index_column] = df_cache[index_column].apply(lambda x: str(x))
-        if save_to_cache:
-            self.df_cache = df_cache
-        return df_cache
-
-    def commit(self):
-        """Commits the graph."""
-        self.graph.commit()
-
-    def get_df(self) -> pd.DataFrame:
-        """Returns the cached dataframe."""
-        if self.df_cache is None:
-            raise ValueError("Cache is empty. Run query_to_dataframe() first with save_to_cache.")
-        return self.df_cache
-
-    def get_instance_properties_from_cache(self, instance_id: str) -> pd.DataFrame:
-        """Returns the properties of an instance."""
-        if self.df_cache is None:
-            raise ValueError("Cache is empty. Run query_to_dataframe() first with save_to_cache.")
-        return self.df_cache.loc[self.df_cache["instance"] == instance_id]
-
-    def print_triples(self):
-        """Prints the triples of the graph."""
-        for subj, pred, obj in self.graph:
-            logging.info(f"Triple: {subj} {pred} {obj}")
-
-    def diagnostic_report(self):
-        """Returns the dictionary representation graph diagnostic data ."""
-        return {
-            "rdf_store_type": self.rdf_store_type,
-            "base_prefix": self.base_prefix,
-            "namespace": self.namespace,
-            "prefixes": self.prefixes,
-            "internal_storage_dir": self.internal_storage_dir,
-            "rdf_store_query_url": self.rdf_store_query_url,
-            "rdf_store_update_url": self.rdf_store_update_url,
-        }
-
-    def add_triples(self, triples: list[Triple] | set[Triple], batch_size: int = 10_000, verbose: bool = False):
+        # All other stores
+        else:
+            if filepath.is_file():
+                self.graph.parse(filepath, publicID=base_uri)
+            else:
+                for filename in filepath.iterdir():
+                    if filename.is_file():
+                        self.graph.parse(filename, publicID=base_uri)
+
+    def _add_triples(self, triples: Iterable[Triple], batch_size: int = 10_000):
         """Adds triples to the graph store in batches.
 
         Args:
@@ -295,27 +174,20 @@ class NeatGraphStoreBase(ABC):
         """
 
         commit_counter = 0
-        if verbose:
-            logging.info(f"Committing total of {len(triples)} triples to knowledge graph!")
-        total_number_of_triples = len(triples)
-        number_of_uploaded_triples = 0
+        number_of_written_triples = 0
 
         def check_commit(force_commit: bool = False):
             """Commit nodes to the graph if batch counter is reached or if force_commit is True"""
             nonlocal commit_counter
-            nonlocal number_of_uploaded_triples
+            nonlocal number_of_written_triples
             if force_commit:
-                number_of_uploaded_triples += commit_counter
+                number_of_written_triples += commit_counter
                 self.graph.commit()
-                if verbose:
-                    logging.info(f"Committed {number_of_uploaded_triples} of {total_number_of_triples} triples")
                 return
             commit_counter += 1
             if commit_counter >= batch_size:
-                number_of_uploaded_triples += commit_counter
+                number_of_written_triples += commit_counter
                 self.graph.commit()
-                if verbose:
-                    logging.info(f"Committed {number_of_uploaded_triples} of {total_number_of_triples} triples")
                 commit_counter = 0
 
         for triple in triples:
@@ -325,25 +197,10 @@ class NeatGraphStoreBase(ABC):
         check_commit(force_commit=True)
 
 
-class _DelayedQuery(Iterable):
-    def __init__(self, graph_ref: Graph, query: str):
-        self.graph_ref = graph_ref
-        self.query = query
-
-    def __iter__(self) -> Iterator[Triple]:
-        start_time = time.perf_counter()
-        result = self.graph_ref.query(self.query)
-        stop_time = time.perf_counter()
-        elapsed_time = stop_time - start_time
-        prom_qsm.labels("query").observe(elapsed_time)
-        prom_sq.labels("query").set(elapsed_time)
-        return cast(Iterator[Triple], iter(result))
-
-
 class _Queries:
     """Helper class for storing standard queries for the graph store."""
 
-    def __init__(self, store: NeatGraphStoreBase):
+    def __init__(self, store: NeatGraphStore):
         self.store = store
 
     def list_instances_ids_of_class(self, class_uri: URIRef, limit: int = -1) -> list[URIRef]:
@@ -359,7 +216,7 @@ class _Queries:
         query_statement = "SELECT DISTINCT ?subject WHERE { ?subject a <class> .} LIMIT X".replace(
             "class", class_uri
         ).replace("LIMIT X", "" if limit == -1 else f"LIMIT {limit}")
-        return [cast(tuple, res)[0] for res in list(self.store.query(query_statement))]
+        return [cast(tuple, res)[0] for res in list(self.store.graph.query(query_statement))]
 
     def list_instances_of_type(self, class_uri: URIRef) -> list[ResultRow]:
         """Get all triples for instances of a given class
@@ -374,20 +231,27 @@ class _Queries:
             f"SELECT ?instance ?prop ?value "
             f"WHERE {{ ?instance rdf:type <{class_uri}> . ?instance ?prop ?value . }} order by ?instance "
         )
-        logging.info(query)
+
         # Select queries gives an iterable of result rows
-        return cast(list[ResultRow], list(self.store.query(query)))
+        return cast(list[ResultRow], list(self.store.graph.query(query)))
 
     def triples_of_type_instances(self, rdf_type: str) -> list[tuple[str, str, str]]:
         """Get all triples of a given type.
 
         This method assumes the graph has been transformed into the default namespace.
         """
-        query = (
-            f"SELECT ?instance ?prop ?value "
-            f"WHERE {{ ?instance a <{self.store.namespace[rdf_type]}> . ?instance ?prop ?value . }} order by ?instance"
-        )
-        result = self.store.query(query)
 
-        # We cannot include the RDF.type in case there is a neat:type property
-        return [remove_namespace(*triple) for triple in result if triple[1] != RDF.type]  # type: ignore[misc, index]
+        if self.store.rules:
+            query = (
+                f"SELECT ?instance ?prop ?value "
+                f"WHERE {{ ?instance a <{self.store.rules.metadata.namespace[rdf_type]}> . ?instance ?prop ?value . }} "
+                "order by ?instance"
+            )
+
+            result = self.store.graph.query(query)
+
+            # We cannot include the RDF.type in case there is a neat:type property
+            return [remove_namespace(*triple) for triple in result if triple[1] != RDF.type]  # type: ignore[misc, index]
+        else:
+            warnings.warn("No rules found for the graph store, returning empty list.", stacklevel=2)
+            return []

cognite/neat/rules/importers/_inference2rules.py

@@ -8,7 +8,7 @@ from rdflib import Literal as RdfLiteral
 
 import cognite.neat.rules.issues as issues
 from cognite.neat.constants import PREFIXES
-from cognite.neat.graph.stores import NeatGraphStoreBase
+from cognite.neat.graph.stores import NeatGraphStore
 from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
 from cognite.neat.rules.issues import IssueList
 from cognite.neat.rules.models import InformationRules, RoleTypes
@@ -47,7 +47,7 @@ class InferenceImporter(BaseImporter):
         self.max_number_of_instance = max_number_of_instance
 
     @classmethod
-    def from_graph_store(cls, store: NeatGraphStoreBase, max_number_of_instance: int = -1):
+    def from_graph_store(cls, store: NeatGraphStore, max_number_of_instance: int = -1):
         issue_list = IssueList(title="Inferred from graph store")
 
         return cls(issue_list, store.graph, max_number_of_instance=max_number_of_instance)

{cognite_neat-0.79.0.dist-info → cognite_neat-0.80.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.79.0
+Version: 0.80.0
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

{cognite_neat-0.79.0.dist-info → cognite_neat-0.80.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 cognite/neat/__init__.py,sha256=v-rRiDOgZ3sQSMQKq0vgUQZvpeOkoHFXissAx6Ktg84,61
-cognite/neat/_version.py,sha256=1nnRuZbUvt4h80F41M2EZqt3TsZYyhJ2C2rRdk9uzhA,23
+cognite/neat/_version.py,sha256=-kfBSEQjpAZbilJOxA4s8jDyP952PRUaVbdaHMuEyIs,23
 cognite/neat/app/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/app/api/asgi/metrics.py,sha256=nxFy7L5cChTI0a-zkCiJ59Aq8yLuIJp5c9Dg0wRXtV0,152
 cognite/neat/app/api/configuration.py,sha256=2U5M6M252swvQPQyooA1EBzFUZNtcTmuSaywfJDgckM,4232
@@ -43,7 +43,8 @@ cognite/neat/app/ui/neat-app/build/static/media/logo.8093b84df9ed36a174c629d6fe0
 cognite/neat/config.py,sha256=oBrWw-KEo0YMbfjVeW6A1FBW7HpP2Pq2ByIq2vlJ10M,6145
 cognite/neat/constants.py,sha256=GYVcrFrvqcznYgB_0jAyykIIiIamxyOpv4dTZWj4K4U,1300
 cognite/neat/exceptions.py,sha256=CM7aCvbek9klOgjTsJ9bfEA8t7KTAL6dc7Mviu4NvSI,4268
-cognite/neat/graph/__init__.py,sha256=31uTeejWOSd-I8iUG8GOZFhHZcQCsBitJ6X8vu2r1nU,73
+cognite/neat/graph/__init__.py,sha256=J8JSJj3s4gFbuAexma__KGpBXPN8wuydPTKd6EwgKPA,65
+cognite/neat/graph/_shared.py,sha256=9QRETdm7hvqIeiHv_n1xi1DUq91Nq7oRRpnPKE0Pnag,181
 cognite/neat/graph/_tracking/__init__.py,sha256=pYj7c-YAUIP4hvN-4mlWnwaeZFerzL9_gM-oZhex7cE,91
 cognite/neat/graph/_tracking/base.py,sha256=8JmaDhlFhSkdBe4SOvFnrdDvMmfTZkHhZxWWWTYkMOQ,820
 cognite/neat/graph/_tracking/log.py,sha256=dBSINd8Tn92hBl8APMD8r6j15g2SlaX1tsDLCmHvaU4,927
@@ -52,7 +53,7 @@ cognite/neat/graph/examples/Knowledge-Graph-Nordic44.xml,sha256=U2Ns-M4LRjT1fBkh
 cognite/neat/graph/examples/__init__.py,sha256=yAjHVY3b5jOjmbW-iLbhvu7BG014TpGi3K4igkDqW5I,368
 cognite/neat/graph/examples/skos-capturing-sheet-wind-topics.xlsx,sha256=CV_yK5ZSbYS_ktfIZUPD8Sevs47zpswLXQUDFkGE4Gw,45798
 cognite/neat/graph/exceptions.py,sha256=R6pyOH774n9w2x_X_nrUr8OMAdjJMf_XPIqAvxIQaWo,3401
-cognite/neat/graph/extractors/__init__.py,sha256=0Mv7iTBwOdMHgqkINh0V2hnDxeC9fIDpBmmnW5Q1lyQ,645
+cognite/neat/graph/extractors/__init__.py,sha256=PRKYPCnxofQ3i_iiJ3xGjEligLgqAPDw2TSlcZt0MlU,947
 cognite/neat/graph/extractors/_base.py,sha256=TOXDnlqske8DgnJwA0THDVRgmR79Acjm56yF0E-2w7I,356
 cognite/neat/graph/extractors/_classic_cdf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/graph/extractors/_classic_cdf/_assets.py,sha256=S5QB_38ysVodGRMqr_SWYYaUtkUCS6a6L2b5D1T-888,3812
@@ -63,20 +64,17 @@ cognite/neat/graph/extractors/_classic_cdf/_relationships.py,sha256=5kClA5zBlhyP
 cognite/neat/graph/extractors/_classic_cdf/_sequences.py,sha256=ov-n8cBEC73AMO1xam2GUDHv-7SyOEWXWRxLXh9flyY,3298
 cognite/neat/graph/extractors/_classic_cdf/_timeseries.py,sha256=xlnJ4fKvCJawZO6l6EHpx36RRAafd3BdYWS0ajNnGVM,4449
 cognite/neat/graph/extractors/_mock_graph_generator.py,sha256=gziG2FFsLk-HmA9uxAeT9RCjVpFxjkCTLiC4tq2zgvw,14961
+cognite/neat/graph/extractors/_rdf_file.py,sha256=w4-XgPgNsmZOkNxjO1ZQCcopTntmmtxfDBkQxn1se6E,463
 cognite/neat/graph/issues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/graph/issues/loader.py,sha256=v8YDsehkUT1QUG61JM9BDV_lqowMUnDmGmbay0aFzN4,3085
 cognite/neat/graph/loaders/__init__.py,sha256=hHC9sfFfbnGSVFTYeuNTIEu4tdLSJ2mWV07fereLelo,125
-cognite/neat/graph/loaders/_base.py,sha256=VOCRIee9ms6FuBlT3mwBV_mQnI6bO53mrardqiMf-Hk,4045
-cognite/neat/graph/loaders/_rdf2dms.py,sha256=bVFLjukCwEUGVoyQ6YnmdRXV945fhX3SiHR6yHLXO2k,12873
+cognite/neat/graph/loaders/_base.py,sha256=bdYC6CwsHVqnQa1QzOhL68qQhF1OtrsearqH6D-z3E4,4037
+cognite/neat/graph/loaders/_rdf2dms.py,sha256=Tn7vy6XwXFXpVDn7uzfzgJMJapbPITerKaF5b5Y4ol4,12857
 cognite/neat/graph/models.py,sha256=AtLgZh2qyRP6NRetjQCy9qLMuTQB0CH52Zsev-qa2sk,149
-cognite/neat/graph/stores/__init__.py,sha256=ivvk7STSo-4wuP_CpizKUCPKmt_ufpNWRJUN9Bv5gdY,543
-cognite/neat/graph/stores/_base.py,sha256=ZeeHxPHV0TU5B3Ep2db6FOkB1VP1gbJzKRjcY46zqGg,14682
-cognite/neat/graph/stores/_graphdb_store.py,sha256=8QM8I4srDKNsR0PddN6htCYUhfkoqlyy-c232Os7C0A,1776
-cognite/neat/graph/stores/_memory_store.py,sha256=GQq19xiyAWU0WQU5txmWnLXBuyP6ywd8plR21UtD3Uw,1420
-cognite/neat/graph/stores/_oxigraph_store.py,sha256=Xj69oE4M-9aqd8bq5CpLCMAhwNjJQAP1AC7lxzDsCn0,5448
+cognite/neat/graph/stores/__init__.py,sha256=G-VG_YwfRt1kuPao07PDJyZ3w_0-eguzLUM13n-Z_RA,64
+cognite/neat/graph/stores/_base.py,sha256=DGmguO0qE5sLHgHG757ymP-cFtEimKvD57Irr3FH9yY,9106
 cognite/neat/graph/stores/_oxrdflib.py,sha256=A5zeRm5_e8ui_ihGpgstRDg_N7qcLZ3QZBRGrOXSGI0,9569
 cognite/neat/graph/stores/_provenance.py,sha256=Y20-I8dP3DwTQ1sdI_eC4va2Az2FpK0oZwdfJ5T-2wc,3279
-cognite/neat/graph/stores/_rdf_to_graph.py,sha256=1ezWHTPn9UkIsAlxZcYRlqWvj3ixlmB5GGG9NN0ls2Q,1244
 cognite/neat/issues.py,sha256=pxQfqfBseMDE8JM0iqZnkLXngeyeFfT0TFtu1UuAd4c,4629
 cognite/neat/legacy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/legacy/graph/__init__.py,sha256=31uTeejWOSd-I8iUG8GOZFhHZcQCsBitJ6X8vu2r1nU,73
@@ -195,7 +193,7 @@ cognite/neat/rules/importers/_dtdl2rules/_unit_lookup.py,sha256=wW4saKva61Q_i17g
 cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py,sha256=ysmWUxZ0npwrTB0uiH5jA0v37sfCwowGaYk17IyxPUU,12663
 cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py,sha256=QDyGt5YBaxzF4v_oCFSgKRSpwVdVruDU3-VW0DEiHbY,6718
 cognite/neat/rules/importers/_dtdl2rules/spec.py,sha256=tim_MfN1J0F3Oeqk3BMgIA82d_MZvhRuRMsLK3B4PYc,11897
-cognite/neat/rules/importers/_inference2rules.py,sha256=JsV3Ii2wmgRELtpV0GC4Y1KtjhyyGR0dtEFpBonHoA8,11213
+cognite/neat/rules/importers/_inference2rules.py,sha256=JgXmhc_6ME9RrA0gVPn9WoVE7NvCrpjdS4_ELW-2e7g,11205
 cognite/neat/rules/importers/_owl2rules/__init__.py,sha256=tdGcrgtozdQyST-pTlxIa4cLBNTLvtk1nNYR4vOdFSw,63
 cognite/neat/rules/importers/_owl2rules/_owl2classes.py,sha256=QpTxvrTGczIa48X8lgXGnMN1AWPhHK0DR6uNq175xak,7357
 cognite/neat/rules/importers/_owl2rules/_owl2metadata.py,sha256=nwnUaBNAAYMoBre2UmsnkJXUuaqGEpR3U3txDrH2w6g,7527
@@ -294,8 +292,8 @@ cognite/neat/workflows/steps_registry.py,sha256=fkTX14ZA7_gkUYfWIlx7A1XbCidvqR23
 cognite/neat/workflows/tasks.py,sha256=dqlJwKAb0jlkl7abbY8RRz3m7MT4SK8-7cntMWkOYjw,788
 cognite/neat/workflows/triggers.py,sha256=_BLNplzoz0iic367u1mhHMHiUrCwP-SLK6_CZzfODX0,7071
 cognite/neat/workflows/utils.py,sha256=gKdy3RLG7ctRhbCRwaDIWpL9Mi98zm56-d4jfHDqP1E,453
-cognite_neat-0.79.0.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-0.79.0.dist-info/METADATA,sha256=Ww02h9AuPpAD5i76P15Q0T3gYFtxd2crDdFGX0yxcBg,9298
-cognite_neat-0.79.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cognite_neat-0.79.0.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
-cognite_neat-0.79.0.dist-info/RECORD,,
+cognite_neat-0.80.0.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-0.80.0.dist-info/METADATA,sha256=dfiX6R98UxzayOm5bMw369QhrIGZmRkwOCIaZdNqxyw,9298
+cognite_neat-0.80.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cognite_neat-0.80.0.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
+cognite_neat-0.80.0.dist-info/RECORD,,

cognite/neat/graph/stores/_graphdb_store.py

@@ -1,51 +0,0 @@
-import logging
-
-import requests
-from rdflib import Graph, Namespace
-from rdflib.plugins.stores.sparqlstore import SPARQLUpdateStore
-
-from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
-
-from ._base import NeatGraphStoreBase
-
-
-class GraphDBStore(NeatGraphStoreBase):
-    """GraphDB is a class that stores the graph in a GraphDB instances and provides methods to
-    read/write data it contains
-
-
-    Args:
-        graph : Instance of rdflib.Graph class for graph storage
-        base_prefix : Used as a base prefix for graph namespace, allowing querying graph data using a shortform of a URI
-        namespace : Namespace (aka URI) used to resolve any relative URI in the graph
-        prefixes : Dictionary of additional prefixes used and bounded to the graph
-    """
-
-    rdf_store_type = "graphdb"
-
-    def __init__(
-        self,
-        graph: Graph | None = None,
-        base_prefix: str = "",  # usually empty
-        namespace: Namespace = DEFAULT_NAMESPACE,
-        prefixes: dict = PREFIXES,
-    ):
-        super().__init__(graph, base_prefix, namespace, prefixes)
-        self.graph_db_rest_url: str = "http://localhost:7200"
-
-    def _set_graph(self) -> None:
-        logging.info("Initializing graph store with GraphDB")
-        store = SPARQLUpdateStore(
-            query_endpoint=self.rdf_store_query_url,
-            update_endpoint=self.rdf_store_update_url,
-            returnFormat=self.returnFormat,
-            context_aware=False,
-            postAsEncoded=False,
-            autocommit=False,
-        )
-        self.graph = Graph(store=store)
-
-    def drop(self):
-        """Drops the graph."""
-        r = requests.delete(f"{self.rdf_store_query_url}/rdf-graphs/service?default")
-        logging.info(f"Dropped graph with state: {r.text}")

cognite/neat/graph/stores/_memory_store.py

@@ -1,43 +0,0 @@
-import logging
-
-from rdflib import Graph, Namespace
-
-from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
-
-from ._base import NeatGraphStoreBase
-
-
-class MemoryStore(NeatGraphStoreBase):
-    """MemoryStore is a class that stores the graph in memory using rdflib and provides
-    methods to read/write data it contains.
-
-
-    Args:
-        graph : Instance of rdflib.Graph class for graph storage
-        base_prefix : Used as a base prefix for graph namespace, allowing querying graph data using a shortform of a URI
-        namespace : Namespace (aka URI) used to resolve any relative URI in the graph
-        prefixes : Dictionary of additional prefixes used and bounded to the graph
-    """
-
-    rdf_store_type: str = "memory"
-
-    def __init__(
-        self,
-        graph: Graph | None = None,
-        base_prefix: str = "",  # usually empty
-        namespace: Namespace = DEFAULT_NAMESPACE,
-        prefixes: dict = PREFIXES,
-    ):
-        # Init repeated to get nice docstring
-        super().__init__(graph, base_prefix, namespace, prefixes)
-
-    def _set_graph(self):
-        logging.info("Initializing graph in memory")
-        self.graph = Graph()
-
-    def drop(self):
-        """Drops the graph."""
-        # In the case of in-memory graph, we just reinitialize the graph
-        # otherwise we would lose the prefixes and bindings, which fails
-        # workflow
-        self.reinitialize_graph()

cognite/neat/graph/stores/_oxigraph_store.py

@@ -1,147 +0,0 @@
-import logging
-import os
-import shutil
-from pathlib import Path
-
-from rdflib import Graph, Namespace
-
-from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
-from cognite.neat.utils.auxiliary import local_import
-
-from ._base import MIMETypes, NeatGraphStoreBase
-
-
-class OxiGraphStore(NeatGraphStoreBase):
-    """OxiGraph is a class that stores the graph using OxiGraph and provides methods to read/write data it contains
-
-
-    Args:
-        graph : Instance of rdflib.Graph class for graph storage
-        base_prefix : Used as a base prefix for graph namespace, allowing querying graph data using a shortform of a URI
-        namespace : Namespace (aka URI) used to resolve any relative URI in the graph
-        prefixes : Dictionary of additional prefixes used and bounded to the graph
-    """
-
-    rdf_store_type = "oxigraph"
-
-    def __init__(
-        self,
-        graph: Graph | None = None,
-        base_prefix: str = "",  # usually empty
-        namespace: Namespace = DEFAULT_NAMESPACE,
-        prefixes: dict = PREFIXES,
-    ):
-        super().__init__(graph, base_prefix, namespace, prefixes)
-
-    def _set_graph(self) -> None:
-        logging.info("Initializing Oxigraph store")
-        local_import("pyoxigraph", "oxi")
-        import pyoxigraph
-
-        from cognite.neat.graph.stores import _oxrdflib
-
-        # Adding support for both in-memory and file-based storage
-        for i in range(4):
-            try:
-                oxstore = pyoxigraph.Store(
-                    path=str(self.internal_storage_dir) if self.internal_storage_dir else None
-                )  # Store (Rust object) accepts only str as path and not Path.
-                break
-            except OSError as e:
-                if "lock" in str(e) and i < 3:
-                    # lock originated from another instance of the store
-                    logging.error("Error initializing Oxigraph store: %s", e)
-                else:
-                    raise e
-        else:
-            raise Exception("Error initializing Oxigraph store")
-
-        self.graph = Graph(store=_oxrdflib.OxigraphStore(store=oxstore))
-        self.graph.default_union = True
-        self.garbage_collector()
-
-    def close(self):
-        """Closes the graph."""
-        if self.graph is not None:
-            try:
-                self.graph.store._inner.flush()  # type: ignore[attr-defined]
-                self.graph.close(True)
-            except Exception as e:
-                logging.debug("Error closing graph: %s", e)
-
-    def restart(self):
-        """Restarts the graph"""
-        self.close()
-        self.reinitialize_graph()
-        logging.info("GraphStore restarted")
-
-    def import_from_file(
-        self, graph_file: Path, mime_type: MIMETypes = "application/rdf+xml", add_base_iri: bool = True
-    ) -> None:
-        """Imports graph data from file.
-
-        Args:
-            graph_file : File path to file containing graph data, by default None
-            mime_type : MIME type of the file, by default "application/rdf+xml"
-            add_base_iri : Add base IRI to the graph, by default True
-        """
-        if add_base_iri:
-            self.graph.store._inner.bulk_load(  # type: ignore[attr-defined]
-                str(graph_file), mime_type, base_iri=self.namespace
-            )
-        else:
-            self.graph.store._inner.bulk_load(str(graph_file), mime_type)  # type: ignore[attr-defined]
-        self.graph.store._inner.optimize()  # type: ignore[attr-defined]
-        return None
-
-    def drop(self):
-        try:
-            self.close()
-            # Due to the specifics of Oxigraph, storage directory cannot be deleted immediately
-            # after closing the graph and creating a new one
-            if self.internal_storage_dir.exists():
-                self.storage_dirs_to_delete.append(self.internal_storage_dir)
-            self.garbage_collector()
-
-        except Exception as e:
-            logging.error(f"Error dropping graph : {e}")
-
-    def garbage_collector(self):
-        """Garbage collection of the graph store."""
-        # delete all directories in self.storage_dirs_to_delete
-        for d in self.storage_dirs_to_delete:
-            shutil.rmtree(d)
-        self.storage_dirs_to_delete = []
-
-    def __del__(self):
-        if self.graph is not None:
-            if self.graph.store is not None:
-                try:
-                    self.graph.store._inner.flush()
-                except Exception:
-                    logging.debug("Error flushing graph")
-            self.graph.close()
-        # It requires more investigation os.remove(self.internal_storage_dir / "LOCK")
-
-    def commit(self):
-        """Commits the graph."""
-        if self.graph:
-            if self.graph.store:
-                logging.info("Committing graph - flushing and optimizing")
-                self.graph.store._inner.flush()
-                self.graph.store._inner.optimize()
-            self.graph.commit()
-
-    @staticmethod
-    def drop_graph_store_storage(storage_path: Path | None) -> None:
-        """Drop graph store storage on disk.
-
-        Args:
-            storage_path : Path to storage directory
-        """
-        if storage_path and storage_path.exists():
-            for f in os.listdir(storage_path):
-                (storage_path / f).unlink()
-            logging.info("Graph store dropped.")
-        else:
-            logging.info(f"Storage path {storage_path} does not exist. Skipping drop.")

cognite/neat/graph/stores/_rdf_to_graph.py

@@ -1,40 +0,0 @@
-from pathlib import Path
-
-from rdflib import Graph, Namespace
-
-from cognite.neat.constants import PREFIXES
-
-
-def rdf_file_to_graph(
-    graph: Graph,
-    filepath: Path,
-    base_prefix: str | None = None,
-    base_namespace: Namespace | None = None,
-    prefixes: dict[str, Namespace] = PREFIXES,
-) -> Graph:
-    """Created rdflib Graph instance loaded with RDF triples from file
-
-    Args:
-        filepath: Path to the RDF file
-        base_prefix: base prefix for URIs. Defaults to None.
-        base_namespace: base namespace for URIs . Defaults to None.
-        prefixes: Dictionary of prefixes to bind to graph. Defaults to PREFIXES.
-        graph: Graph instance to load RDF triples into. Defaults to None.
-
-    Returns:
-        Graph instance loaded with RDF triples from file
-    """
-
-    if filepath.is_file():
-        graph.parse(filepath, publicID=base_namespace)
-    else:
-        for filename in filepath.iterdir():
-            if filename.is_file():
-                graph.parse(filename, publicID=base_namespace)
-    if base_prefix and base_namespace:
-        graph.bind(base_prefix, base_namespace)
-    if prefixes:
-        for prefix, namespace in prefixes.items():
-            graph.bind(prefix, namespace)
-
-    return graph