cognite-neat 0.78.5__tar.gz → 0.80.0__tar.gz

{cognite_neat-0.78.5 → cognite_neat-0.80.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.78.5
+Version: 0.80.0
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.80.0/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.80.0"

cognite_neat-0.80.0/cognite/neat/graph/__init__.py

@@ -0,0 +1,3 @@
+from .stores import NeatGraphStore
+
+__all__ = ["NeatGraphStore"]

cognite_neat-0.80.0/cognite/neat/graph/_shared.py

@@ -0,0 +1,5 @@
+from typing import Literal, TypeAlias
+
+MIMETypes: TypeAlias = Literal[
+    "application/rdf+xml", "text/turtle", "application/n-triple", "application/n-quads", "application/trig"
+]

{cognite_neat-0.78.5 → cognite_neat-0.80.0}/cognite/neat/graph/extractors/__init__.py

@@ -6,6 +6,7 @@ from ._classic_cdf._relationships import RelationshipsExtractor
 from ._classic_cdf._sequences import SequencesExtractor
 from ._classic_cdf._timeseries import TimeSeriesExtractor
 from ._mock_graph_generator import MockGraphGenerator
+from ._rdf_file import RdfFileExtractor
 
 __all__ = [
     "AssetsExtractor",
@@ -16,4 +17,18 @@ __all__ = [
     "EventsExtractor",
     "FilesExtractor",
     "LabelsExtractor",
+    "RdfFileExtractor",
 ]
+
+
+TripleExtractors = (
+    AssetsExtractor
+    | MockGraphGenerator
+    | RelationshipsExtractor
+    | TimeSeriesExtractor
+    | SequencesExtractor
+    | EventsExtractor
+    | FilesExtractor
+    | LabelsExtractor
+    | RdfFileExtractor
+)

cognite_neat-0.80.0/cognite/neat/graph/extractors/_rdf_file.py

@@ -0,0 +1,18 @@
+from pathlib import Path
+
+from rdflib import URIRef
+
+from cognite.neat.graph._shared import MIMETypes
+from cognite.neat.graph.extractors._base import BaseExtractor
+
+
+class RdfFileExtractor(BaseExtractor):
+    def __init__(
+        self,
+        filepath: Path,
+        mime_type: MIMETypes = "application/rdf+xml",
+        base_uri: URIRef | None = None,
+    ):
+        self.filepath = filepath
+        self.mime_type = mime_type
+        self.base_uri = base_uri

{cognite_neat-0.78.5 → cognite_neat-0.80.0}/cognite/neat/graph/loaders/_base.py

@@ -6,7 +6,7 @@ from typing import ClassVar, Generic, Literal, TypeVar, overload
 from cognite.client import CogniteClient
 from cognite.client.data_classes.capabilities import Capability
 
-from cognite.neat.graph import NeatGraphStoreBase
+from cognite.neat.graph import NeatGraphStore
 from cognite.neat.graph.issues.loader import FailedAuthorizationError
 from cognite.neat.issues import NeatIssue, NeatIssueList
 from cognite.neat.utils.upload import UploadDiffsID, UploadResultIDs
@@ -18,7 +18,7 @@ class BaseLoader(ABC, Generic[T_Output]):
     _new_line = "\n"
     _encoding = "utf-8"
 
-    def __init__(self, graph_store: NeatGraphStoreBase):
+    def __init__(self, graph_store: NeatGraphStore):
         self.graph_store = graph_store
 
     @abstractmethod

{cognite_neat-0.78.5 → cognite_neat-0.80.0}/cognite/neat/graph/loaders/_rdf2dms.py

@@ -17,7 +17,7 @@ from pydantic.main import Model
 
 from cognite.neat.graph._tracking import LogTracker, Tracker
 from cognite.neat.graph.issues import loader as loader_issues
-from cognite.neat.graph.stores import NeatGraphStoreBase
+from cognite.neat.graph.stores import NeatGraphStore
 from cognite.neat.issues import NeatIssue, NeatIssueList
 from cognite.neat.rules.models import DMSRules
 from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE
@@ -29,7 +29,7 @@ from ._base import CDFLoader
 class DMSLoader(CDFLoader[dm.InstanceApply]):
     def __init__(
         self,
-        graph_store: NeatGraphStoreBase,
+        graph_store: NeatGraphStore,
         data_model: dm.DataModel[dm.View] | None,
         instance_space: str,
         class_by_view_id: dict[ViewId, str] | None = None,
@@ -48,7 +48,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         cls,
         client: CogniteClient,
         data_model_id: dm.DataModelId,
-        graph_store: NeatGraphStoreBase,
+        graph_store: NeatGraphStore,
         instance_space: str,
     ) -> "DMSLoader":
         issues: list[NeatIssue] = []
@@ -61,7 +61,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         return cls(graph_store, data_model, instance_space, {}, issues)
 
     @classmethod
-    def from_rules(cls, rules: DMSRules, graph_store: NeatGraphStoreBase, instance_space: str) -> "DMSLoader":
+    def from_rules(cls, rules: DMSRules, graph_store: NeatGraphStore, instance_space: str) -> "DMSLoader":
         issues: list[NeatIssue] = []
         data_model: dm.DataModel[dm.View] | None = None
         try:

cognite_neat-0.80.0/cognite/neat/graph/stores/__init__.py

@@ -0,0 +1,3 @@
+from ._base import NeatGraphStore
+
+__all__ = ["NeatGraphStore"]

cognite_neat-0.80.0/cognite/neat/graph/stores/_base.py

@@ -0,0 +1,257 @@
+import sys
+import warnings
+from collections.abc import Iterable
+from datetime import datetime
+from pathlib import Path
+from typing import cast
+
+import pytz
+from rdflib import RDF, Graph, Namespace, URIRef
+from rdflib.plugins.stores.sparqlstore import SPARQLUpdateStore
+from rdflib.query import ResultRow
+
+from cognite.neat.graph._shared import MIMETypes
+from cognite.neat.graph.extractors import RdfFileExtractor, TripleExtractors
+from cognite.neat.graph.models import Triple
+from cognite.neat.rules.models.information import InformationRules
+from cognite.neat.utils import remove_namespace
+from cognite.neat.utils.auxiliary import local_import
+
+from ._provenance import Change, Provenance
+
+if sys.version_info < (3, 11):
+    from typing_extensions import Self
+else:
+    from typing import Self
+
+
+class NeatGraphStore:
+    """NeatGraphStore is a class that stores the graph and provides methods to read/write data it contains
+
+
+    Args:
+        graph : Instance of rdflib.Graph class for graph storage
+        rules:
+    """
+
+    rdf_store_type: str
+
+    def __init__(
+        self,
+        graph: Graph,
+        rules: InformationRules | None = None,
+    ):
+        _start = datetime.now(pytz.utc)
+        self.graph = graph
+        self.provenance = Provenance(
+            [
+                Change.record(
+                    activity=f"{type(self).__name__}.__init__",
+                    start=_start,
+                    end=datetime.now(pytz.utc),
+                    description=f"Initialize graph store as {type(self.graph.store).__name__}",
+                )
+            ]
+        )
+        self.rules = rules
+
+        if self.rules and self.rules.prefixes:
+            self._upsert_prefixes(self.rules.prefixes)
+
+        self.queries = _Queries(self)
+
+    def _upsert_prefixes(self, prefixes: dict[str, Namespace]) -> None:
+        """Adds prefixes to the graph store."""
+        _start = datetime.now(pytz.utc)
+        for prefix, namespace in prefixes.items():
+            self.graph.bind(prefix, namespace)
+
+        self.provenance.append(
+            Change.record(
+                activity=f"{type(self).__name__}._upsert_prefixes",
+                start=_start,
+                end=datetime.now(pytz.utc),
+                description="Upsert prefixes to graph store",
+            )
+        )
+
+    @classmethod
+    def from_memory_store(cls, rules: InformationRules | None = None) -> "Self":
+        return cls(Graph(), rules)
+
+    @classmethod
+    def from_sparql_store(
+        cls,
+        query_endpoint: str | None = None,
+        update_endpoint: str | None = None,
+        returnFormat: str = "csv",
+        rules: InformationRules | None = None,
+    ) -> "Self":
+        store = SPARQLUpdateStore(
+            query_endpoint=query_endpoint,
+            update_endpoint=update_endpoint,
+            returnFormat=returnFormat,
+            context_aware=False,
+            postAsEncoded=False,
+            autocommit=False,
+        )
+        graph = Graph(store=store)
+        return cls(graph, rules)
+
+    @classmethod
+    def from_oxi_store(cls, storage_dir: Path | None = None, rules: InformationRules | None = None) -> "Self":
+        """Creates a NeatGraphStore from an Oxigraph store."""
+        local_import("pyoxigraph", "oxi")
+        import pyoxigraph
+
+        from cognite.neat.graph.stores._oxrdflib import OxigraphStore
+
+        # Adding support for both oxigraph in-memory and file-based storage
+        for i in range(4):
+            try:
+                oxi_store = pyoxigraph.Store(path=str(storage_dir) if storage_dir else None)
+                break
+            except OSError as e:
+                if "lock" in str(e) and i < 3:
+                    continue
+                raise e
+        else:
+            raise Exception("Error initializing Oxigraph store")
+
+        graph = Graph(store=OxigraphStore(store=oxi_store))
+        graph.default_union = True
+
+        return cls(graph, rules)
+
+    def write(self, extractor: TripleExtractors) -> None:
+        if isinstance(extractor, RdfFileExtractor):
+            self._parse_file(extractor.filepath, extractor.mime_type, extractor.base_uri)
+        else:
+            self._add_triples(extractor.extract())
+
+    def _parse_file(
+        self,
+        filepath: Path,
+        mime_type: MIMETypes = "application/rdf+xml",
+        base_uri: URIRef | None = None,
+    ) -> None:
+        """Imports graph data from file.
+
+        Args:
+            filepath : File path to file containing graph data, by default None
+            mime_type : MIME type of graph data, by default "application/rdf+xml"
+            add_base_iri : Add base IRI to graph, by default True
+        """
+
+        # Oxigraph store, do not want to type hint this as it is an optional dependency
+        if type(self.graph.store).__name__ == "OxigraphStore":
+
+            def parse_to_oxi_store():
+                local_import("pyoxigraph", "oxi")
+                from cognite.neat.graph.stores._oxrdflib import OxigraphStore
+
+                cast(OxigraphStore, self.graph.store)._inner.bulk_load(str(filepath), mime_type, base_iri=base_uri)  # type: ignore[attr-defined]
+                cast(OxigraphStore, self.graph.store)._inner.optimize()  # type: ignore[attr-defined]
+
+            parse_to_oxi_store()
+
+        # All other stores
+        else:
+            if filepath.is_file():
+                self.graph.parse(filepath, publicID=base_uri)
+            else:
+                for filename in filepath.iterdir():
+                    if filename.is_file():
+                        self.graph.parse(filename, publicID=base_uri)
+
+    def _add_triples(self, triples: Iterable[Triple], batch_size: int = 10_000):
+        """Adds triples to the graph store in batches.
+
+        Args:
+            triples: list of triples to be added to the graph store
+            batch_size: Batch size of triples per commit, by default 10_000
+            verbose: Verbose mode, by default False
+        """
+
+        commit_counter = 0
+        number_of_written_triples = 0
+
+        def check_commit(force_commit: bool = False):
+            """Commit nodes to the graph if batch counter is reached or if force_commit is True"""
+            nonlocal commit_counter
+            nonlocal number_of_written_triples
+            if force_commit:
+                number_of_written_triples += commit_counter
+                self.graph.commit()
+                return
+            commit_counter += 1
+            if commit_counter >= batch_size:
+                number_of_written_triples += commit_counter
+                self.graph.commit()
+                commit_counter = 0
+
+        for triple in triples:
+            self.graph.add(triple)
+            check_commit()
+
+        check_commit(force_commit=True)
+
+
+class _Queries:
+    """Helper class for storing standard queries for the graph store."""
+
+    def __init__(self, store: NeatGraphStore):
+        self.store = store
+
+    def list_instances_ids_of_class(self, class_uri: URIRef, limit: int = -1) -> list[URIRef]:
+        """Get instances ids for a given class
+
+        Args:
+            class_uri: Class for which instances are to be found
+            limit: Max number of instances to return, by default -1 meaning all instances
+
+        Returns:
+            List of class instance URIs
+        """
+        query_statement = "SELECT DISTINCT ?subject WHERE { ?subject a <class> .} LIMIT X".replace(
+            "class", class_uri
+        ).replace("LIMIT X", "" if limit == -1 else f"LIMIT {limit}")
+        return [cast(tuple, res)[0] for res in list(self.store.graph.query(query_statement))]
+
+    def list_instances_of_type(self, class_uri: URIRef) -> list[ResultRow]:
+        """Get all triples for instances of a given class
+
+        Args:
+            class_uri: Class for which instances are to be found
+
+        Returns:
+            List of triples for instances of the given class
+        """
+        query = (
+            f"SELECT ?instance ?prop ?value "
+            f"WHERE {{ ?instance rdf:type <{class_uri}> . ?instance ?prop ?value . }} order by ?instance "
+        )
+
+        # Select queries gives an iterable of result rows
+        return cast(list[ResultRow], list(self.store.graph.query(query)))
+
+    def triples_of_type_instances(self, rdf_type: str) -> list[tuple[str, str, str]]:
+        """Get all triples of a given type.
+
+        This method assumes the graph has been transformed into the default namespace.
+        """
+
+        if self.store.rules:
+            query = (
+                f"SELECT ?instance ?prop ?value "
+                f"WHERE {{ ?instance a <{self.store.rules.metadata.namespace[rdf_type]}> . ?instance ?prop ?value . }} "
+                "order by ?instance"
+            )
+
+            result = self.store.graph.query(query)
+
+            # We cannot include the RDF.type in case there is a neat:type property
+            return [remove_namespace(*triple) for triple in result if triple[1] != RDF.type]  # type: ignore[misc, index]
+        else:
+            warnings.warn("No rules found for the graph store, returning empty list.", stacklevel=2)
+            return []

cognite_neat-0.80.0/cognite/neat/graph/stores/_provenance.py

@@ -0,0 +1,99 @@
+# we will use prov-o to represent the provenance of the neat graph store
+# basically tracking changes that occur in the graph store
+# prov-o use concepts of Agent, Activity and Entity to represent provenance
+# where in case of neat we have:
+# Agent: triples extractors, graph enhancers, contextualizers, etc.
+# Activity: write triple, add connection, etc.
+# Entity: neat graph store
+
+
+import uuid
+from collections import UserList
+from collections.abc import Sequence
+from dataclasses import dataclass
+from datetime import datetime
+from typing import TypeVar
+
+from rdflib import PROV, RDF, Literal, URIRef
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+
+
+@dataclass(frozen=True)
+class Agent:
+    id_: URIRef = DEFAULT_NAMESPACE.agent
+    acted_on_behalf_of: str = "NEAT"
+
+    def as_triples(self):
+        return [
+            (self.id_, RDF.type, PROV[type(self).__name__]),
+            (self.id_, PROV.actedOnBehalfOf, self.acted_on_behalf_of),
+        ]
+
+
+@dataclass(frozen=True)
+class Activity:
+    was_associated_with: Agent
+    ended_at_time: datetime
+    started_at_time: datetime
+    used: str  # this would be set to for example Extractor, Enhancer, Contextualizer, etc.
+    id_: URIRef = DEFAULT_NAMESPACE[f"activity-{uuid.uuid4()}"]
+
+    def as_triples(self):
+        return [
+            (self.id_, RDF.type, PROV[type(self).__name__]),
+            (self.id_, PROV.wasAssociatedWith, self.was_associated_with.id_),
+            (self.id_, PROV.startedAtTime, Literal(self.started_at_time)),
+            (self.id_, PROV.endedAtTime, Literal(self.ended_at_time)),
+            (self.id_, PROV.used, self.used),
+        ]
+
+
+@dataclass(frozen=True)
+class Entity:
+    was_generated_by: Activity
+    was_attributed_to: Agent
+    id_: URIRef = DEFAULT_NAMESPACE["graph-store"]
+
+    def as_triples(self):
+        return [
+            (self.id_, RDF.type, PROV[type(self).__name__]),
+            (self.id_, PROV.wasGeneratedBy, self.was_generated_by.id_),
+            (self.id_, PROV.wasAttributedTo, self.was_attributed_to.id_),
+        ]
+
+
+@dataclass(frozen=True)
+class Change:
+    agent: Agent
+    activity: Activity
+    entity: Entity
+    description: str
+
+    def as_triples(self):
+        return self.agent.as_triples() + self.activity.as_triples() + self.entity.as_triples()
+
+    @classmethod
+    def record(cls, activity: str, start: datetime, end: datetime, description: str):
+        """User friendly method to record a change that occurred in the graph store."""
+        agent = Agent()
+        activity = Activity(used=activity, was_associated_with=agent, started_at_time=start, ended_at_time=end)
+        entity = Entity(was_generated_by=activity, was_attributed_to=agent)
+        return cls(agent, activity, entity, description)
+
+
+T_Change = TypeVar("T_Change", bound=Change)
+
+
+class Provenance(UserList[T_Change]):
+    def __init__(self, changes: Sequence[T_Change] | None = None):
+        super().__init__(changes or [])
+
+    def did_this_happen(self, this: str) -> bool:
+        return any(change.description == this for change in self)
+
+    def __delitem__(self, *args, **kwargs):
+        raise TypeError("Cannot delete change from provenance")
+
+    def __setitem__(self, *args, **kwargs):
+        raise TypeError("Cannot modify change from provenance")

{cognite_neat-0.78.5 → cognite_neat-0.80.0}/cognite/neat/rules/importers/_inference2rules.py

@@ -8,7 +8,7 @@ from rdflib import Literal as RdfLiteral
 
 import cognite.neat.rules.issues as issues
 from cognite.neat.constants import PREFIXES
-from cognite.neat.graph.stores import NeatGraphStoreBase
+from cognite.neat.graph.stores import NeatGraphStore
 from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
 from cognite.neat.rules.issues import IssueList
 from cognite.neat.rules.models import InformationRules, RoleTypes
@@ -47,7 +47,7 @@ class InferenceImporter(BaseImporter):
         self.max_number_of_instance = max_number_of_instance
 
     @classmethod
-    def from_graph_store(cls, store: NeatGraphStoreBase, max_number_of_instance: int = -1):
+    def from_graph_store(cls, store: NeatGraphStore, max_number_of_instance: int = -1):
         issue_list = IssueList(title="Inferred from graph store")
 
         return cls(issue_list, store.graph, max_number_of_instance=max_number_of_instance)

{cognite_neat-0.78.5 → cognite_neat-0.80.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.78.5"
+version = "0.80.0"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.78.5/cognite/neat/_version.py

@@ -1 +0,0 @@
-__version__ = "0.78.5"