trustgraph-flow 0.20.11__tar.gz → 0.21.1__tar.gz

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.20.11
+Version: 0.21.1
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.20.11.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.21.1.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

trustgraph-flow-0.21.1/scripts/librarian

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.librarian import run
+
+run()
+

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/setup.py

@@ -34,7 +34,7 @@ setuptools.setup(
     python_requires='>=3.8',
     download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
     install_requires=[
-        "trustgraph-base>=0.20,<0.21",
+        "trustgraph-base>=0.21,<0.22",
         "aiohttp",
         "anthropic",
         "cassandra-driver",
@@ -49,6 +49,7 @@ setuptools.setup(
         "langchain-community",
         "langchain-core",
         "langchain-text-splitters",
+        "minio",
         "neo4j",
         "ollama",
         "openai",
@@ -78,8 +79,8 @@ setuptools.setup(
         "scripts/de-write-qdrant",
         "scripts/document-embeddings",
         "scripts/document-rag",
-        "scripts/embeddings-ollama",
         "scripts/embeddings-fastembed",
+        "scripts/embeddings-ollama",
         "scripts/ge-query-milvus",
         "scripts/ge-query-pinecone",
         "scripts/ge-query-qdrant",
@@ -91,6 +92,7 @@ setuptools.setup(
         "scripts/kg-extract-definitions",
         "scripts/kg-extract-relationships",
         "scripts/kg-extract-topics",
+        "scripts/librarian",
         "scripts/metering",
         "scripts/object-extract-row",
         "scripts/oe-write-milvus",

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/agent/react/service.py

@@ -189,7 +189,7 @@ class Processor(ConsumerProducer):
 
         return json.loads(json_str)
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         try:
 
@@ -229,7 +229,7 @@ class Processor(ConsumerProducer):
                     observation=None,
                 )
 
-                self.producer.send(r, properties={"id": id})
+                await self.producer.send(r, properties={"id": id})
 
             def observe(x):
 
@@ -242,7 +242,7 @@ class Processor(ConsumerProducer):
                     observation=x,
                 )
 
-                self.producer.send(r, properties={"id": id})
+                await self.producer.send(r, properties={"id": id})
 
             act = self.agent.react(v.question, history, think, observe)
 
@@ -258,7 +258,7 @@ class Processor(ConsumerProducer):
                     thought=None,
                 )
 
-                self.producer.send(r, properties={"id": id})
+                await self.producer.send(r, properties={"id": id})
 
                 print("Done.", flush=True)
 
@@ -281,7 +281,7 @@ class Processor(ConsumerProducer):
                 ]
             )
 
-            self.recursive_input.send(r, properties={"id": id})
+            await self.recursive_input.send(r, properties={"id": id})
 
             print("Done.", flush=True)
 
@@ -301,7 +301,7 @@ class Processor(ConsumerProducer):
                 response=None,
             )
 
-            self.producer.send(r, properties={"id": id})
+            await self.producer.send(r, properties={"id": id})
 
     @staticmethod
     def add_args(parser):
@@ -377,5 +377,5 @@ description.'''
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/chunking/recursive/chunker.py

@@ -52,7 +52,7 @@ class Processor(ConsumerProducer):
             is_separator_regex=False,
         )
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Chunking {v.metadata.id}...", flush=True)
@@ -70,7 +70,7 @@ class Processor(ConsumerProducer):
 
             __class__.chunk_metric.observe(len(chunk.page_content))
 
-            self.send(r)
+            await self.send(r)
 
         print("Done.", flush=True)
 
@@ -98,5 +98,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/chunking/token/chunker.py

@@ -51,7 +51,7 @@ class Processor(ConsumerProducer):
             chunk_overlap=chunk_overlap,
         )
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Chunking {v.metadata.id}...", flush=True)
@@ -69,7 +69,7 @@ class Processor(ConsumerProducer):
 
             __class__.chunk_metric.observe(len(chunk.page_content))
 
-            self.send(r)
+            await self.send(r)
 
         print("Done.", flush=True)
 
@@ -97,5 +97,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/decoding/pdf/pdf_decoder.py

@@ -39,7 +39,7 @@ class Processor(ConsumerProducer):
 
         print("PDF inited")
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         print("PDF message received")
 
@@ -64,7 +64,7 @@ class Processor(ConsumerProducer):
                         text=page.page_content.encode("utf-8"),
                     )
 
-                    self.send(r)
+                    await self.send(r)
 
         print("Done.", flush=True)
 
@@ -78,5 +78,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/direct/cassandra.py

@@ -6,7 +6,7 @@ class TrustGraph:
 
     def __init__(
             self, hosts=None,
-            keyspace="trustgraph", table="default",
+            keyspace="trustgraph", table="default", username=None, password=None
     ):
 
         if hosts is None:
@@ -14,8 +14,13 @@ class TrustGraph:
 
         self.keyspace = keyspace
         self.table = table
+        self.username = username
             
-        self.cluster = Cluster(hosts)
+        if username and password:
+            auth_provider = PlainTextAuthProvider(username=username, password=password)
+            self.cluster = Cluster(hosts, auth_provider=auth_provider)
+        else:
+            self.cluster = Cluster(hosts)
         self.session = self.cluster.connect()
 
         self.init()

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/embeddings/document_embeddings/embeddings.py

@@ -52,7 +52,7 @@ class Processor(ConsumerProducer):
             subscriber=module + "-emb",
         )
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Indexing {v.metadata.id}...", flush=True)
@@ -73,7 +73,7 @@ class Processor(ConsumerProducer):
                 chunks=embeds,
             )
 
-            self.producer.send(r)
+            await self.send(r)
 
         except Exception as e:
             print("Exception:", e, flush=True)
@@ -105,5 +105,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/embeddings/fastembed/processor.py

@@ -41,7 +41,7 @@ class Processor(ConsumerProducer):
 
         self.embeddings = TextEmbedding(model_name = model)
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
 
@@ -65,7 +65,7 @@ class Processor(ConsumerProducer):
             error=None,
         )
 
-        self.producer.send(r, properties={"id": id})
+        await self.producer.send(r, properties={"id": id})
 
         print("Done.", flush=True)
 
@@ -85,5 +85,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/embeddings/graph_embeddings/embeddings.py

@@ -52,7 +52,7 @@ class Processor(ConsumerProducer):
             subscriber=module + "-emb",
         )
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Indexing {v.metadata.id}...", flush=True)
@@ -77,7 +77,7 @@ class Processor(ConsumerProducer):
                 entities=entities,
             )
 
-            self.producer.send(r)
+            await self.send(r)
 
         except Exception as e:
             print("Exception:", e, flush=True)
@@ -109,5 +109,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/embeddings/ollama/processor.py

@@ -45,7 +45,7 @@ class Processor(ConsumerProducer):
         self.client = Client(host=ollama)
         self.model = model
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
 
@@ -67,7 +67,7 @@ class Processor(ConsumerProducer):
             error=None,
         )
 
-        self.producer.send(r, properties={"id": id})
+        await self.producer.send(r, properties={"id": id})
 
         print("Done.", flush=True)
 
@@ -93,5 +93,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/external/wikipedia/service.py

@@ -39,7 +39,7 @@ class Processor(ConsumerProducer):
 
         self.url = url
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
 
@@ -60,7 +60,7 @@ class Processor(ConsumerProducer):
                 text=resp
             )
 
-            self.producer.send(r, properties={"id": id})
+            await self.producer.send(r, properties={"id": id})
 
             self.consumer.acknowledge(msg)
 
@@ -75,7 +75,7 @@ class Processor(ConsumerProducer):
                 ),
                 text=None,
             )
-            self.producer.send(r, properties={"id": id})
+            await self.producer.send(r, properties={"id": id})
 
             self.consumer.acknowledge(msg)
 
@@ -98,5 +98,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/extract/kg/definitions/extract.py

@@ -96,15 +96,15 @@ class Processor(ConsumerProducer):
 
         return self.prompt.request_definitions(chunk)
 
-    def emit_edges(self, metadata, triples):
+    async def emit_edges(self, metadata, triples):
 
         t = Triples(
             metadata=metadata,
             triples=triples,
         )
-        self.producer.send(t)
+        await self.send(t)
 
-    def emit_ecs(self, metadata, entities):
+    async def emit_ecs(self, metadata, entities):
 
         t = EntityContexts(
             metadata=metadata,
@@ -112,7 +112,7 @@ class Processor(ConsumerProducer):
         )
         self.ec_prod.send(t)
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Indexing {v.metadata.id}...", flush=True)
@@ -171,7 +171,7 @@ class Processor(ConsumerProducer):
                 entities.append(ec)
                     
 
-            self.emit_edges(
+            await self.emit_edges(
                 Metadata(
                     id=v.metadata.id,
                     metadata=[],
@@ -181,7 +181,7 @@ class Processor(ConsumerProducer):
                 triples
             )
 
-            self.emit_ecs(
+            await self.emit_ecs(
                 Metadata(
                     id=v.metadata.id,
                     metadata=[],
@@ -224,5 +224,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/extract/kg/relationships/extract.py

@@ -71,15 +71,15 @@ class Processor(ConsumerProducer):
 
         return self.prompt.request_relationships(chunk)
 
-    def emit_edges(self, metadata, triples):
+    async def emit_edges(self, metadata, triples):
 
         t = Triples(
             metadata=metadata,
             triples=triples,
         )
-        self.producer.send(t)
+        await self.send(t)
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Indexing {v.metadata.id}...", flush=True)
@@ -166,7 +166,7 @@ class Processor(ConsumerProducer):
                         o=Value(value=v.metadata.id, is_uri=True)
                     ))
 
-            self.emit_edges(
+            await self.emit_edges(
                 Metadata(
                     id=v.metadata.id,
                     metadata=[],
@@ -203,5 +203,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/extract/kg/topics/extract.py

@@ -69,15 +69,15 @@ class Processor(ConsumerProducer):
 
         return self.prompt.request_topics(chunk)
 
-    def emit_edge(self, metadata, s, p, o):
+    async def emit_edge(self, metadata, s, p, o):
 
         t = Triples(
             metadata=metadata,
             triples=[Triple(s=s, p=p, o=o)],
         )
-        self.producer.send(t)
+        await self.send(t)
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Indexing {v.metadata.id}...", flush=True)
@@ -104,7 +104,9 @@ class Processor(ConsumerProducer):
                 s_value = Value(value=str(s_uri), is_uri=True)
                 o_value = Value(value=str(o), is_uri=False)
 
-                self.emit_edge(v. metadata, s_value, DEFINITION_VALUE, o_value)
+                await self.emit_edge(
+                    v.metadata, s_value, DEFINITION_VALUE, o_value
+                )
 
         except Exception as e:
             print("Exception: ", e, flush=True)
@@ -133,5 +135,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/extract/object/row/extract.py

@@ -129,16 +129,16 @@ class Processor(ConsumerProducer):
         t = Rows(
             metadata=metadata, row_schema=self.row_schema, rows=rows
         )
-        self.producer.send(t)
+        await self.producer.send(t)
 
     def emit_vec(self, metadata, name, vec, key_name, key):
 
         r = ObjectEmbeddings(
             metadata=metadata, vectors=vec, name=name, key_name=key_name, id=key
         )
-        self.vec_prod.send(r)
+        await self.vec_prod.send(r)
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Indexing {v.metadata.id}...", flush=True)
@@ -216,5 +216,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

trustgraph-flow-0.21.1/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "0.21.1"

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/gateway/document_embeddings_load.py

@@ -7,8 +7,8 @@ from aiohttp import WSMsgType
 from .. schema import Metadata
 from .. schema import DocumentEmbeddings, ChunkEmbeddings
 from .. schema import document_embeddings_store_queue
+from .. base import Publisher
 
-from . publisher import Publisher
 from . socket import SocketEndpoint
 from . serialize import to_subgraph
 
@@ -59,6 +59,6 @@ class DocumentEmbeddingsLoadEndpoint(SocketEndpoint):
                     ],
                 )
 
-                self.publisher.send(None, elt)
+                await self.publisher.send(None, elt)
 
         running.stop()

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/gateway/document_embeddings_stream.py

@@ -6,8 +6,8 @@ import uuid
 
 from .. schema import DocumentEmbeddings
 from .. schema import document_embeddings_store_queue
+from .. base import Subscriber
 
-from . subscriber import Subscriber
 from . socket import SocketEndpoint
 from . serialize import serialize_document_embeddings
 

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/gateway/endpoint.py

@@ -5,8 +5,8 @@ from aiohttp import web
 import uuid
 import logging
 
-from . publisher import Publisher
-from . subscriber import Subscriber
+from .. base import Publisher
+from .. base import Subscriber
 
 logger = logging.getLogger("endpoint")
 logger.setLevel(logging.INFO)

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/gateway/graph_embeddings_load.py

@@ -7,8 +7,8 @@ from aiohttp import WSMsgType
 from .. schema import Metadata
 from .. schema import GraphEmbeddings, EntityEmbeddings
 from .. schema import graph_embeddings_store_queue
+from .. base import Publisher
 
-from . publisher import Publisher
 from . socket import SocketEndpoint
 from . serialize import to_subgraph, to_value
 
@@ -59,7 +59,7 @@ class GraphEmbeddingsLoadEndpoint(SocketEndpoint):
                     ]
                 )
 
-                self.publisher.send(None, elt)
+                await self.publisher.send(None, elt)
 
 
         running.stop()

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/gateway/graph_embeddings_stream.py

@@ -6,8 +6,8 @@ import uuid
 
 from .. schema import GraphEmbeddings
 from .. schema import graph_embeddings_store_queue
+from .. base import Subscriber
 
-from . subscriber import Subscriber
 from . socket import SocketEndpoint
 from . serialize import serialize_graph_embeddings
 

trustgraph-flow-0.21.1/trustgraph/gateway/librarian.py

@@ -0,0 +1,57 @@
+
+from .. schema import LibrarianRequest, LibrarianResponse, Triples
+from .. schema import librarian_request_queue
+from .. schema import librarian_response_queue
+
+from . endpoint import ServiceEndpoint
+from . requestor import ServiceRequestor
+from . serialize import serialize_document_package, serialize_document_info
+from . serialize import to_document_package, to_document_info, to_criteria
+
+class LibrarianRequestor(ServiceRequestor):
+    def __init__(self, pulsar_host, timeout, auth):
+
+        super(LibrarianRequestor, self).__init__(
+            pulsar_host=pulsar_host,
+            request_queue=librarian_request_queue,
+            response_queue=librarian_response_queue,
+            request_schema=LibrarianRequest,
+            response_schema=LibrarianResponse,
+            timeout=timeout,
+        )
+
+    def to_request(self, body):
+
+        if "document" in body:
+            dp = to_document_package(body["document"])
+        else:
+            dp = None
+
+        if "criteria" in body:
+            criteria = to_criteria(body["criteria"])
+        else:
+            criteria = None
+
+        limit = int(body.get("limit", 10000))
+
+        return LibrarianRequest(
+            operation = body.get("operation", None),
+            id = body.get("id", None),
+            document = dp,
+            user = body.get("user", None),
+            collection = body.get("collection", None),
+            criteria = criteria,
+        )
+
+    def from_response(self, message):
+
+        response = {}
+
+        if message.document:
+            response["document"] = serialize_document_package(message.document)
+
+        if message.info:
+            response["info"] = serialize_document_info(message.info)
+        
+        return response, True
+

{trustgraph-flow-0.20.11 → trustgraph-flow-0.21.1}/trustgraph/gateway/requestor.py

@@ -4,8 +4,8 @@ from pulsar.schema import JsonSchema
 import uuid
 import logging
 
-from . publisher import Publisher
-from . subscriber import Subscriber
+from .. base import Publisher
+from .. base import Subscriber
 
 logger = logging.getLogger("requestor")
 logger.setLevel(logging.INFO)
@@ -68,7 +68,10 @@ class ServiceRequestor:
                     raise RuntimeError("Timeout")
 
                 if resp.error:
-                    err = { "error": resp.error.message }
+                    err = { "error": {
+                        "type": resp.error.type,
+                        "message": resp.error.message,
+                    } }
                     if responder:
                         await responder(err, True)
                     return err
@@ -87,7 +90,10 @@ class ServiceRequestor:
 
             logging.error(f"Exception: {e}")
 
-            err = { "error": str(e) }
+            err = { "error": {
+                "type": "gateway-error",
+                "message": str(e),
+            } }
             if responder:
                 await responder(err, True)
             return err