trustgraph-flow 2.7.6__tar.gz → 2.7.8__tar.gz

{trustgraph_flow-2.7.6 → trustgraph_flow-2.7.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-flow
-Version: 2.7.6
+Version: 2.7.8
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

{trustgraph_flow-2.7.6 → trustgraph_flow-2.7.8}/trustgraph/cores/knowledge.py

@@ -15,11 +15,24 @@ import logging
 # Module logger
 logger = logging.getLogger(__name__)
 
+
+async def _librarian_error(respond, message):
+    await respond(
+        KnowledgeResponse(
+            error=Error(
+                type="librarian-error",
+                message=message,
+            ),
+        )
+    )
+
+
 class KnowledgeManager:
 
     def __init__(
             self, cassandra_host, cassandra_username, cassandra_password,
-            keyspace, flow_config, librarian=None, replication_factor=1,
+            keyspace, flow_config, librarian_clients=None,
+            replication_factor=1,
     ):
 
         self.table_store = KnowledgeTableStore(
@@ -27,7 +40,7 @@ class KnowledgeManager:
             replication_factor
         )
 
-        self.librarian = librarian
+        self.librarian_clients = librarian_clients if librarian_clients is not None else {}
         self._pending_library_metadata = {}
 
         self.loader_queue = asyncio.Queue(maxsize=20)
@@ -90,8 +103,19 @@ class KnowledgeManager:
             publish_ge,
         )
 
-        if self.librarian:
-            await self._stream_library_docs(request.id, respond)
+        librarian = self.librarian_clients.get(workspace)
+        if librarian is None:
+            logger.error(
+                f"No librarian client for workspace {workspace}"
+            )
+            await _librarian_error(
+                respond,
+                f"No librarian client for workspace {workspace}",
+            )
+        else:
+            await self._stream_library_docs(
+                librarian, request.id, respond,
+            )
 
         logger.debug("Knowledge core retrieval complete")
 
@@ -129,11 +153,22 @@ class KnowledgeManager:
                 workspace, request.graph_embeddings
             )
 
-        if request.library_metadata and self.librarian:
-            await self._put_library_metadata(request.library_metadata, workspace)
+        librarian = self.librarian_clients.get(workspace)
 
-        if request.library_blob and self.librarian:
-            await self._put_library_blob(request.library_blob, workspace)
+        if request.library_metadata or request.library_blob:
+            if librarian is None:
+                logger.error(
+                    f"No librarian client for workspace {workspace}"
+                )
+            else:
+                if request.library_metadata:
+                    await self._put_library_metadata(
+                        request.library_metadata, workspace,
+                    )
+                if request.library_blob:
+                    await self._put_library_blob(
+                        librarian, request.library_blob, workspace,
+                    )
 
         await respond(
             KnowledgeResponse(
@@ -263,36 +298,42 @@ class KnowledgeManager:
 
         await self.loader_queue.put((request, respond, workspace))
 
-    async def _stream_library_docs(self, document_id, respond):
+    async def _stream_library_docs(self, librarian, document_id, respond):
 
         try:
-            root_meta = await self.librarian.fetch_document_metadata(
+            root_meta = await librarian.fetch_document_metadata(
                 document_id
             )
         except Exception as e:
             logger.warning(f"Could not fetch library metadata for {document_id}: {e}")
+            await _librarian_error(respond, str(e))
             return
 
         if root_meta is None:
             return
 
-        await self._stream_one_doc(root_meta, respond)
+        await self._stream_doc_tree(librarian, root_meta, respond)
+
+    async def _stream_doc_tree(self, librarian, doc_meta, respond):
+
+        await self._stream_one_doc(librarian, doc_meta, respond)
 
         try:
-            resp = await self.librarian.request(
+            resp = await librarian.request(
                 LibrarianRequest(
                     operation="list-children",
-                    document_id=document_id,
+                    document_id=doc_meta.id,
                 )
             )
         except Exception as e:
-            logger.warning(f"Could not list children for {document_id}: {e}")
+            logger.warning(f"Could not list children for {doc_meta.id}: {e}")
+            await _librarian_error(respond, str(e))
             return
 
         for child_meta in resp.document_metadatas:
-            await self._stream_one_doc(child_meta, respond)
+            await self._stream_doc_tree(librarian, child_meta, respond)
 
-    async def _stream_one_doc(self, doc_meta, respond):
+    async def _stream_one_doc(self, librarian, doc_meta, respond):
 
         lm = LibraryMetadata(
             id=doc_meta.id,
@@ -309,11 +350,12 @@ class KnowledgeManager:
         )
 
         try:
-            content = await self.librarian.fetch_document_content(
+            content = await librarian.fetch_document_content(
                 doc_meta.id
             )
         except Exception as e:
             logger.warning(f"Could not fetch content for {doc_meta.id}: {e}")
+            await _librarian_error(respond, str(e))
             return
 
         await respond(
@@ -328,7 +370,7 @@ class KnowledgeManager:
     async def _put_library_metadata(self, lm, workspace):
         self._pending_library_metadata[lm.id] = lm
 
-    async def _put_library_blob(self, lb, workspace):
+    async def _put_library_blob(self, librarian, lb, workspace):
 
         lm = self._pending_library_metadata.pop(lb.id, None)
         if lm is None:
@@ -353,7 +395,7 @@ class KnowledgeManager:
             operation = "add-document"
 
         try:
-            await self.librarian.request(
+            await librarian.request(
                 LibrarianRequest(
                     operation=operation,
                     document_id=lm.id,

{trustgraph_flow-2.7.6 → trustgraph_flow-2.7.8}/trustgraph/cores/service.py

@@ -16,6 +16,7 @@ from .. base import LibrarianClient
 
 from .. schema import KnowledgeRequest, KnowledgeResponse, Error
 from .. schema import knowledge_request_queue, knowledge_response_queue
+from .. schema import librarian_request_queue, librarian_response_queue
 
 from .. schema import Document, Metadata
 from .. schema import TextDocument, Metadata
@@ -79,9 +80,7 @@ class Processor(WorkspaceProcessor):
             }
         )
 
-        self.librarian_client = LibrarianClient(
-            id=id, backend=self.pubsub, taskgroup=self.taskgroup,
-        )
+        self.librarian_clients = {}
 
         self.knowledge = KnowledgeManager(
             cassandra_host = self.cassandra_host,
@@ -89,7 +88,7 @@ class Processor(WorkspaceProcessor):
             cassandra_password = self.cassandra_password,
             keyspace = keyspace,
             flow_config = self,
-            librarian = self.librarian_client,
+            librarian_clients = self.librarian_clients,
             replication_factor = replication_factor,
         )
 
@@ -142,18 +141,39 @@ class Processor(WorkspaceProcessor):
             ),
         )
 
+        librarian_client = LibrarianClient(
+            id=self.id,
+            backend=self.pubsub,
+            taskgroup=self.taskgroup,
+            librarian_request_queue=workspace_queue(
+                librarian_request_queue, workspace,
+            ),
+            librarian_response_queue=workspace_queue(
+                librarian_response_queue, workspace,
+            ),
+            librarian_subscriber=(
+                f"{self.id}--{workspace}--librarian"
+            ),
+        )
+
         await response_producer.start()
         await consumer.start()
+        await librarian_client.start()
+
+        self.librarian_clients[workspace] = librarian_client
 
         self.workspace_consumers[workspace] = {
             "consumer": consumer,
             "response": response_producer,
+            "librarian": librarian_client,
         }
 
         logger.info(f"Subscribed to workspace queue: {workspace}")
 
     async def on_workspace_deleted(self, workspace):
 
+        self.librarian_clients.pop(workspace, None)
+
         clients = self.workspace_consumers.pop(workspace, None)
         if clients:
             for client in clients.values():
@@ -163,7 +183,6 @@ class Processor(WorkspaceProcessor):
     async def start(self):
 
         await super(Processor, self).start()
-        await self.librarian_client.start()
 
     async def on_knowledge_config(self, workspace, config, version):
 

trustgraph_flow-2.7.8/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "2.7.8"

{trustgraph_flow-2.7.6 → trustgraph_flow-2.7.8}/trustgraph/tables/knowledge.py

@@ -1,18 +1,92 @@
 
+import json
+
 from .. schema import KnowledgeResponse, Triple, Triples, EntityEmbeddings
-from .. schema import Metadata, Term, IRI, LITERAL, GraphEmbeddings
+from .. schema import Metadata, Term, IRI, BLANK, LITERAL, TRIPLE
+from .. schema import GraphEmbeddings
 from .. schema import DocumentEmbeddings, ChunkEmbeddings
 
 from cassandra.cluster import Cluster
 
 from . cassandra_async import async_execute, async_execute_paged
 
+TRIPLE_MARKER = "<<TRIPLE>>"
+BLANK_MARKER = "<<BLANK>>"
+
+
+def _serialize_term(term):
+    if term.type == IRI:
+        return {"t": "i", "i": term.iri}
+    elif term.type == LITERAL:
+        r = {"t": "l", "v": term.value}
+        if term.datatype:
+            r["dt"] = term.datatype
+        if term.language:
+            r["ln"] = term.language
+        return r
+    elif term.type == BLANK:
+        return {"t": "b", "d": term.id}
+    elif term.type == TRIPLE and term.triple:
+        return {"t": "t", "tr": _serialize_triple(term.triple)}
+    return {"t": term.type}
+
+
+def _deserialize_term(data):
+    t = data.get("t", "")
+    if t == "i":
+        return Term(type=IRI, iri=data.get("i", ""))
+    elif t == "l":
+        return Term(
+            type=LITERAL, value=data.get("v", ""),
+            datatype=data.get("dt", ""), language=data.get("ln", ""),
+        )
+    elif t == "b":
+        return Term(type=BLANK, id=data.get("d", ""))
+    elif t == "t":
+        tr = data.get("tr")
+        return Term(
+            type=TRIPLE,
+            triple=_deserialize_triple(tr) if tr else None,
+        )
+    return Term(type=t)
+
+
+def _serialize_triple(triple):
+    r = {}
+    if triple.s:
+        r["s"] = _serialize_term(triple.s)
+    if triple.p:
+        r["p"] = _serialize_term(triple.p)
+    if triple.o:
+        r["o"] = _serialize_term(triple.o)
+    if triple.g:
+        r["g"] = triple.g
+    return r
+
+
+def _deserialize_triple(data):
+    return Triple(
+        s=_deserialize_term(data["s"]) if "s" in data else None,
+        p=_deserialize_term(data["p"]) if "p" in data else None,
+        o=_deserialize_term(data["o"]) if "o" in data else None,
+        g=data.get("g"),
+    )
+
 
 def term_to_tuple(term):
     """Convert Term to (value, is_uri) tuple for database storage."""
     if term.type == IRI:
         return (term.iri, True)
-    else:  # LITERAL
+    elif term.type == TRIPLE:
+        return (
+            TRIPLE_MARKER + json.dumps(
+                _serialize_triple(term.triple), separators=(",", ":")
+            ),
+            False,
+        )
+    elif term.type == BLANK:
+        return (BLANK_MARKER + term.id, False)
+    else:
         return (term.value, False)
 
 
@@ -20,6 +94,13 @@ def tuple_to_term(value, is_uri):
     """Convert (value, is_uri) tuple from database to Term."""
     if is_uri:
         return Term(type=IRI, iri=value)
+    elif value.startswith(TRIPLE_MARKER):
+        triple_data = json.loads(value[len(TRIPLE_MARKER):])
+        return Term(
+            type=TRIPLE, triple=_deserialize_triple(triple_data),
+        )
+    elif value.startswith(BLANK_MARKER):
+        return Term(type=BLANK, id=value[len(BLANK_MARKER):])
     else:
         return Term(type=LITERAL, value=value)
 from cassandra.auth import PlainTextAuthProvider

{trustgraph_flow-2.7.6 → trustgraph_flow-2.7.8}/trustgraph_flow.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-flow
-Version: 2.7.6
+Version: 2.7.8
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

trustgraph_flow-2.7.6/trustgraph/flow_version.py

@@ -1 +0,0 @@
-__version__ = "2.7.6"