trustgraph-flow 0.21.0__tar.gz → 0.21.2__tar.gz

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.21.0
+Version: 0.21.2
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.21.0.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.21.2.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

trustgraph-flow-0.21.2/scripts/librarian

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.librarian import run
+
+run()
+

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/setup.py

@@ -49,6 +49,7 @@ setuptools.setup(
         "langchain-community",
         "langchain-core",
         "langchain-text-splitters",
+        "minio",
         "neo4j",
         "ollama",
         "openai",
@@ -78,8 +79,8 @@ setuptools.setup(
         "scripts/de-write-qdrant",
         "scripts/document-embeddings",
         "scripts/document-rag",
-        "scripts/embeddings-ollama",
         "scripts/embeddings-fastembed",
+        "scripts/embeddings-ollama",
         "scripts/ge-query-milvus",
         "scripts/ge-query-pinecone",
         "scripts/ge-query-qdrant",
@@ -91,6 +92,7 @@ setuptools.setup(
         "scripts/kg-extract-definitions",
         "scripts/kg-extract-relationships",
         "scripts/kg-extract-topics",
+        "scripts/librarian",
         "scripts/metering",
         "scripts/object-extract-row",
         "scripts/oe-write-milvus",

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/agent/react/service.py

@@ -189,7 +189,7 @@ class Processor(ConsumerProducer):
 
         return json.loads(json_str)
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         try:
 
@@ -229,7 +229,7 @@ class Processor(ConsumerProducer):
                     observation=None,
                 )
 
-                self.producer.send(r, properties={"id": id})
+                await self.producer.send(r, properties={"id": id})
 
             def observe(x):
 
@@ -242,7 +242,7 @@ class Processor(ConsumerProducer):
                     observation=x,
                 )
 
-                self.producer.send(r, properties={"id": id})
+                await self.producer.send(r, properties={"id": id})
 
             act = self.agent.react(v.question, history, think, observe)
 
@@ -258,7 +258,7 @@ class Processor(ConsumerProducer):
                     thought=None,
                 )
 
-                self.producer.send(r, properties={"id": id})
+                await self.producer.send(r, properties={"id": id})
 
                 print("Done.", flush=True)
 
@@ -281,7 +281,7 @@ class Processor(ConsumerProducer):
                 ]
             )
 
-            self.recursive_input.send(r, properties={"id": id})
+            await self.recursive_input.send(r, properties={"id": id})
 
             print("Done.", flush=True)
 
@@ -301,7 +301,7 @@ class Processor(ConsumerProducer):
                 response=None,
             )
 
-            self.producer.send(r, properties={"id": id})
+            await self.producer.send(r, properties={"id": id})
 
     @staticmethod
     def add_args(parser):
@@ -377,5 +377,5 @@ description.'''
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/chunking/recursive/chunker.py

@@ -52,7 +52,7 @@ class Processor(ConsumerProducer):
             is_separator_regex=False,
         )
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Chunking {v.metadata.id}...", flush=True)
@@ -70,7 +70,7 @@ class Processor(ConsumerProducer):
 
             __class__.chunk_metric.observe(len(chunk.page_content))
 
-            self.send(r)
+            await self.send(r)
 
         print("Done.", flush=True)
 
@@ -98,5 +98,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/chunking/token/chunker.py

@@ -51,7 +51,7 @@ class Processor(ConsumerProducer):
             chunk_overlap=chunk_overlap,
         )
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Chunking {v.metadata.id}...", flush=True)
@@ -69,7 +69,7 @@ class Processor(ConsumerProducer):
 
             __class__.chunk_metric.observe(len(chunk.page_content))
 
-            self.send(r)
+            await self.send(r)
 
         print("Done.", flush=True)
 
@@ -97,5 +97,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/decoding/pdf/pdf_decoder.py

@@ -39,7 +39,7 @@ class Processor(ConsumerProducer):
 
         print("PDF inited")
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         print("PDF message received")
 
@@ -64,7 +64,7 @@ class Processor(ConsumerProducer):
                         text=page.page_content.encode("utf-8"),
                     )
 
-                    self.send(r)
+                    await self.send(r)
 
         print("Done.", flush=True)
 
@@ -78,5 +78,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/embeddings/document_embeddings/embeddings.py

@@ -52,7 +52,7 @@ class Processor(ConsumerProducer):
             subscriber=module + "-emb",
         )
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Indexing {v.metadata.id}...", flush=True)
@@ -73,7 +73,7 @@ class Processor(ConsumerProducer):
                 chunks=embeds,
             )
 
-            self.producer.send(r)
+            await self.send(r)
 
         except Exception as e:
             print("Exception:", e, flush=True)
@@ -105,5 +105,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/embeddings/fastembed/processor.py

@@ -41,7 +41,7 @@ class Processor(ConsumerProducer):
 
         self.embeddings = TextEmbedding(model_name = model)
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
 
@@ -65,7 +65,7 @@ class Processor(ConsumerProducer):
             error=None,
         )
 
-        self.producer.send(r, properties={"id": id})
+        await self.producer.send(r, properties={"id": id})
 
         print("Done.", flush=True)
 
@@ -85,5 +85,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/embeddings/graph_embeddings/embeddings.py

@@ -52,7 +52,7 @@ class Processor(ConsumerProducer):
             subscriber=module + "-emb",
         )
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Indexing {v.metadata.id}...", flush=True)
@@ -77,7 +77,7 @@ class Processor(ConsumerProducer):
                 entities=entities,
             )
 
-            self.producer.send(r)
+            await self.send(r)
 
         except Exception as e:
             print("Exception:", e, flush=True)
@@ -109,5 +109,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/embeddings/ollama/processor.py

@@ -45,7 +45,7 @@ class Processor(ConsumerProducer):
         self.client = Client(host=ollama)
         self.model = model
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
 
@@ -67,7 +67,7 @@ class Processor(ConsumerProducer):
             error=None,
         )
 
-        self.producer.send(r, properties={"id": id})
+        await self.producer.send(r, properties={"id": id})
 
         print("Done.", flush=True)
 
@@ -93,5 +93,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/external/wikipedia/service.py

@@ -39,7 +39,7 @@ class Processor(ConsumerProducer):
 
         self.url = url
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
 
@@ -60,7 +60,7 @@ class Processor(ConsumerProducer):
                 text=resp
             )
 
-            self.producer.send(r, properties={"id": id})
+            await self.producer.send(r, properties={"id": id})
 
             self.consumer.acknowledge(msg)
 
@@ -75,7 +75,7 @@ class Processor(ConsumerProducer):
                 ),
                 text=None,
             )
-            self.producer.send(r, properties={"id": id})
+            await self.producer.send(r, properties={"id": id})
 
             self.consumer.acknowledge(msg)
 
@@ -98,5 +98,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/extract/kg/definitions/extract.py

@@ -96,15 +96,15 @@ class Processor(ConsumerProducer):
 
         return self.prompt.request_definitions(chunk)
 
-    def emit_edges(self, metadata, triples):
+    async def emit_edges(self, metadata, triples):
 
         t = Triples(
             metadata=metadata,
             triples=triples,
         )
-        self.producer.send(t)
+        await self.send(t)
 
-    def emit_ecs(self, metadata, entities):
+    async def emit_ecs(self, metadata, entities):
 
         t = EntityContexts(
             metadata=metadata,
@@ -112,7 +112,7 @@ class Processor(ConsumerProducer):
         )
         self.ec_prod.send(t)
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Indexing {v.metadata.id}...", flush=True)
@@ -171,7 +171,7 @@ class Processor(ConsumerProducer):
                 entities.append(ec)
                     
 
-            self.emit_edges(
+            await self.emit_edges(
                 Metadata(
                     id=v.metadata.id,
                     metadata=[],
@@ -181,7 +181,7 @@ class Processor(ConsumerProducer):
                 triples
             )
 
-            self.emit_ecs(
+            await self.emit_ecs(
                 Metadata(
                     id=v.metadata.id,
                     metadata=[],
@@ -224,5 +224,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/extract/kg/relationships/extract.py

@@ -71,15 +71,15 @@ class Processor(ConsumerProducer):
 
         return self.prompt.request_relationships(chunk)
 
-    def emit_edges(self, metadata, triples):
+    async def emit_edges(self, metadata, triples):
 
         t = Triples(
             metadata=metadata,
             triples=triples,
         )
-        self.producer.send(t)
+        await self.send(t)
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Indexing {v.metadata.id}...", flush=True)
@@ -166,7 +166,7 @@ class Processor(ConsumerProducer):
                         o=Value(value=v.metadata.id, is_uri=True)
                     ))
 
-            self.emit_edges(
+            await self.emit_edges(
                 Metadata(
                     id=v.metadata.id,
                     metadata=[],
@@ -203,5 +203,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/extract/kg/topics/extract.py

@@ -69,15 +69,15 @@ class Processor(ConsumerProducer):
 
         return self.prompt.request_topics(chunk)
 
-    def emit_edge(self, metadata, s, p, o):
+    async def emit_edge(self, metadata, s, p, o):
 
         t = Triples(
             metadata=metadata,
             triples=[Triple(s=s, p=p, o=o)],
         )
-        self.producer.send(t)
+        await self.send(t)
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Indexing {v.metadata.id}...", flush=True)
@@ -104,7 +104,9 @@ class Processor(ConsumerProducer):
                 s_value = Value(value=str(s_uri), is_uri=True)
                 o_value = Value(value=str(o), is_uri=False)
 
-                self.emit_edge(v. metadata, s_value, DEFINITION_VALUE, o_value)
+                await self.emit_edge(
+                    v.metadata, s_value, DEFINITION_VALUE, o_value
+                )
 
         except Exception as e:
             print("Exception: ", e, flush=True)
@@ -133,5 +135,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/extract/object/row/extract.py

@@ -129,16 +129,16 @@ class Processor(ConsumerProducer):
         t = Rows(
             metadata=metadata, row_schema=self.row_schema, rows=rows
         )
-        self.producer.send(t)
+        await self.producer.send(t)
 
     def emit_vec(self, metadata, name, vec, key_name, key):
 
         r = ObjectEmbeddings(
             metadata=metadata, vectors=vec, name=name, key_name=key_name, id=key
         )
-        self.vec_prod.send(r)
+        await self.vec_prod.send(r)
 
-    def handle(self, msg):
+    async def handle(self, msg):
 
         v = msg.value()
         print(f"Indexing {v.metadata.id}...", flush=True)
@@ -216,5 +216,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

trustgraph-flow-0.21.2/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "0.21.2"

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/gateway/agent.py

@@ -7,10 +7,10 @@ from . endpoint import ServiceEndpoint
 from . requestor import ServiceRequestor
 
 class AgentRequestor(ServiceRequestor):
-    def __init__(self, pulsar_host, timeout, auth):
+    def __init__(self, pulsar_client, timeout, auth):
 
         super(AgentRequestor, self).__init__(
-            pulsar_host=pulsar_host,
+            pulsar_client=pulsar_client,
             request_queue=agent_request_queue,
             response_queue=agent_response_queue,
             request_schema=AgentRequest,

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/gateway/dbpedia.py

@@ -7,10 +7,10 @@ from . endpoint import ServiceEndpoint
 from . requestor import ServiceRequestor
 
 class DbpediaRequestor(ServiceRequestor):
-    def __init__(self, pulsar_host, timeout, auth):
+    def __init__(self, pulsar_client, timeout, auth):
 
         super(DbpediaRequestor, self).__init__(
-            pulsar_host=pulsar_host,
+            pulsar_client=pulsar_client,
             request_queue=dbpedia_lookup_request_queue,
             response_queue=dbpedia_lookup_response_queue,
             request_schema=LookupRequest,

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/gateway/document_embeddings_load.py

@@ -7,25 +7,25 @@ from aiohttp import WSMsgType
 from .. schema import Metadata
 from .. schema import DocumentEmbeddings, ChunkEmbeddings
 from .. schema import document_embeddings_store_queue
+from .. base import Publisher
 
-from . publisher import Publisher
 from . socket import SocketEndpoint
 from . serialize import to_subgraph
 
 class DocumentEmbeddingsLoadEndpoint(SocketEndpoint):
 
     def __init__(
-            self, pulsar_host, auth, path="/api/v1/load/document-embeddings",
+            self, pulsar_client, auth, path="/api/v1/load/document-embeddings",
     ):
 
         super(DocumentEmbeddingsLoadEndpoint, self).__init__(
             endpoint_path=path, auth=auth,
         )
 
-        self.pulsar_host=pulsar_host
+        self.pulsar_client=pulsar_client
 
         self.publisher = Publisher(
-            self.pulsar_host, document_embeddings_store_queue,
+            self.pulsar_client, document_embeddings_store_queue,
             schema=JsonSchema(DocumentEmbeddings)
         )
 
@@ -59,6 +59,6 @@ class DocumentEmbeddingsLoadEndpoint(SocketEndpoint):
                     ],
                 )
 
-                self.publisher.send(None, elt)
+                await self.publisher.send(None, elt)
 
         running.stop()

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/gateway/document_embeddings_stream.py

@@ -6,25 +6,26 @@ import uuid
 
 from .. schema import DocumentEmbeddings
 from .. schema import document_embeddings_store_queue
+from .. base import Subscriber
 
-from . subscriber import Subscriber
 from . socket import SocketEndpoint
 from . serialize import serialize_document_embeddings
 
 class DocumentEmbeddingsStreamEndpoint(SocketEndpoint):
 
     def __init__(
-            self, pulsar_host, auth, path="/api/v1/stream/document-embeddings"
+            self, pulsar_client, auth,
+            path="/api/v1/stream/document-embeddings"
     ):
 
         super(DocumentEmbeddingsStreamEndpoint, self).__init__(
             endpoint_path=path, auth=auth,
         )
 
-        self.pulsar_host=pulsar_host
+        self.pulsar_client=pulsar_client
 
         self.subscriber = Subscriber(
-            self.pulsar_host, document_embeddings_store_queue,
+            self.pulsar_client, document_embeddings_store_queue,
             "api-gateway", "api-gateway",
             schema=JsonSchema(DocumentEmbeddings)
         )

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/gateway/document_load.py

@@ -8,10 +8,10 @@ from . sender import ServiceSender
 from . serialize import to_subgraph
 
 class DocumentLoadSender(ServiceSender):
-    def __init__(self, pulsar_host):
+    def __init__(self, pulsar_client):
 
         super(DocumentLoadSender, self).__init__(
-            pulsar_host=pulsar_host,
+            pulsar_client=pulsar_client,
             request_queue=document_ingest_queue,
             request_schema=Document,
         )

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/gateway/document_rag.py

@@ -7,10 +7,10 @@ from . endpoint import ServiceEndpoint
 from . requestor import ServiceRequestor
 
 class DocumentRagRequestor(ServiceRequestor):
-    def __init__(self, pulsar_host, timeout, auth):
+    def __init__(self, pulsar_client, timeout, auth):
 
         super(DocumentRagRequestor, self).__init__(
-            pulsar_host=pulsar_host,
+            pulsar_client=pulsar_client,
             request_queue=document_rag_request_queue,
             response_queue=document_rag_response_queue,
             request_schema=DocumentRagQuery,

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/gateway/embeddings.py

@@ -7,10 +7,10 @@ from . endpoint import ServiceEndpoint
 from . requestor import ServiceRequestor
 
 class EmbeddingsRequestor(ServiceRequestor):
-    def __init__(self, pulsar_host, timeout, auth):
+    def __init__(self, pulsar_client, timeout, auth):
 
         super(EmbeddingsRequestor, self).__init__(
-            pulsar_host=pulsar_host,
+            pulsar_client=pulsar_client,
             request_queue=embeddings_request_queue,
             response_queue=embeddings_response_queue,
             request_schema=EmbeddingsRequest,

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/gateway/encyclopedia.py

@@ -7,10 +7,10 @@ from . endpoint import ServiceEndpoint
 from . requestor import ServiceRequestor
 
 class EncyclopediaRequestor(ServiceRequestor):
-    def __init__(self, pulsar_host, timeout, auth):
+    def __init__(self, pulsar_client, timeout, auth):
 
         super(EncyclopediaRequestor, self).__init__(
-            pulsar_host=pulsar_host,
+            pulsar_client=pulsar_client,
             request_queue=encyclopedia_lookup_request_queue,
             response_queue=encyclopedia_lookup_response_queue,
             request_schema=LookupRequest,

{trustgraph-flow-0.21.0 → trustgraph-flow-0.21.2}/trustgraph/gateway/endpoint.py

@@ -5,8 +5,8 @@ from aiohttp import web
 import uuid
 import logging
 
-from . publisher import Publisher
-from . subscriber import Subscriber
+from .. base import Publisher
+from .. base import Subscriber
 
 logger = logging.getLogger("endpoint")
 logger.setLevel(logging.INFO)