trustgraph-flow 0.17.1__tar.gz → 0.17.3__tar.gz

{trustgraph-flow-0.17.1 → trustgraph-flow-0.17.3}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.17.1
+Version: 0.17.3
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 License: UNKNOWN
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.17.1.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.17.3.tar.gz
 Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)

{trustgraph-flow-0.17.1 → trustgraph-flow-0.17.3}/trustgraph/api/gateway/endpoint.py

@@ -41,10 +41,15 @@ class ServiceEndpoint:
 
         self.operation = "service"
 
-    async def start(self):
+    async def start(self, client):
 
-        self.pub_task = asyncio.create_task(self.pub.run())
-        self.sub_task = asyncio.create_task(self.sub.run())
+        self.pub_task = asyncio.create_task(self.pub.run(client))
+        self.sub_task = asyncio.create_task(self.sub.run(client))
+
+    async def join(self):
+        
+        await self.pub_task
+        await self.sub_task
 
     def add_routes(self, app):
 
@@ -62,6 +67,8 @@ class ServiceEndpoint:
 
         id = str(uuid.uuid4())
 
+        print(request.path, "...")
+
         try:
             ht = request.headers["Authorization"]
             tokens = ht.split(" ", 2)
@@ -78,23 +85,31 @@ class ServiceEndpoint:
 
             data = await request.json()
 
+            print(data)
+
             q = await self.sub.subscribe(id)
 
             await self.pub.send(
                 id,
                 self.to_request(data),
             )
+            print("Request sent")
 
             try:
                 resp = await asyncio.wait_for(q.get(), self.timeout)
             except:
                 raise RuntimeError("Timeout waiting for response")
 
+            print("Response got")
+
             if resp.error:
+                print("Error")
                 return web.json_response(
                     { "error": resp.error.message }
                 )
 
+            print("Send response")
+
             return web.json_response(
                 self.from_response(resp)
             )

{trustgraph-flow-0.17.1 → trustgraph-flow-0.17.3}/trustgraph/api/gateway/graph_embeddings_load.py

@@ -29,10 +29,10 @@ class GraphEmbeddingsLoadEndpoint(SocketEndpoint):
             schema=JsonSchema(GraphEmbeddings)
         )
 
-    async def start(self):
+    async def start(self, client):
 
         self.task = asyncio.create_task(
-            self.publisher.run()
+            self.publisher.run(client)
         )
 
     async def listener(self, ws, running):

{trustgraph-flow-0.17.1 → trustgraph-flow-0.17.3}/trustgraph/api/gateway/graph_embeddings_stream.py

@@ -28,10 +28,10 @@ class GraphEmbeddingsStreamEndpoint(SocketEndpoint):
             schema=JsonSchema(GraphEmbeddings)
         )
 
-    async def start(self):
+    async def start(self, client):
 
         self.task = asyncio.create_task(
-            self.subscriber.run()
+            self.subscriber.run(client)
         )
 
     async def async_thread(self, ws, running):

trustgraph-flow-0.17.3/trustgraph/api/gateway/publisher.py

@@ -0,0 +1,39 @@
+
+import asyncio
+
+class Publisher:
+
+    def __init__(self, pulsar_host, topic, schema=None, max_size=10,
+                 chunking_enabled=False):
+        self.pulsar_host = pulsar_host
+        self.topic = topic
+        self.schema = schema
+        self.q = asyncio.Queue(maxsize=max_size)
+        self.chunking_enabled = chunking_enabled
+
+    async def run(self, client):
+
+        while True:
+
+            try:
+                async with client.create_producer(
+                        topic=self.topic,
+                        schema=self.schema,
+                        chunking_enabled=self.chunking_enabled,
+                ) as producer:
+                    while True:
+                        id, item = await self.q.get()
+
+                        if id:
+                            await producer.send(item, { "id": id })
+                        else:
+                            await producer.send(item)
+
+            except Exception as e:
+                print("Exception:", e, flush=True)
+
+            # If handler drops out, sleep a retry
+            await asyncio.sleep(2)
+
+    async def send(self, id, msg):
+        await self.q.put((id, msg))

{trustgraph-flow-0.17.1 → trustgraph-flow-0.17.3}/trustgraph/api/gateway/service.py

@@ -17,6 +17,7 @@ from aiohttp import web
 import logging
 import os
 import base64
+import aiopulsar
 
 import pulsar
 from pulsar.schema import JsonSchema
@@ -237,13 +238,35 @@ class Api:
                 { "error": str(e) }
             )
 
-    async def app_factory(self):
+    async def run_endpoints(self):
 
-        for ep in self.endpoints:
-            await ep.start()
+        async with aiopulsar.connect(self.pulsar_host) as client:
+
+            for ep in self.endpoints:
+                await ep.start(client)
+
+                self.doc_ingest_pub_task = asyncio.create_task(
+                    self.document_out.run(client)
+                )
+
+                self.text_ingest_pub_task = asyncio.create_task(
+                    self.text_out.run(client)
+                )
+
+            print("Endpoints are running...")
+
+            # They never exit
+            for ep in self.endpoints:
+                await ep.join()
+
+            await self.doc_ingest_pub_task
+            await self.text_ingest_pub_task
+
+            print("Endpoints are stopped.")
+
+    async def app_factory(self):
 
-        self.doc_ingest_pub_task = asyncio.create_task(self.document_out.run())
-        self.text_ingest_pub_task = asyncio.create_task(self.text_out.run())
+        self.endpoint_task = asyncio.create_task(self.run_endpoints())
 
         return self.app
 

{trustgraph-flow-0.17.1 → trustgraph-flow-0.17.3}/trustgraph/api/gateway/socket.py

@@ -76,6 +76,12 @@ class SocketEndpoint:
     async def start(self):
         pass
 
+    async def join(self):
+
+        # Nothing to wait for
+        while True:
+            await asyncio.sleep(100)
+
     def add_routes(self, app):
 
         app.add_routes([

trustgraph-flow-0.17.3/trustgraph/api/gateway/subscriber.py

@@ -0,0 +1,66 @@
+
+import asyncio
+
+class Subscriber:
+
+    def __init__(self, pulsar_host, topic, subscription, consumer_name,
+                 schema=None, max_size=10):
+        self.pulsar_host = pulsar_host
+        self.topic = topic
+        self.subscription = subscription
+        self.consumer_name = consumer_name
+        self.schema = schema
+        self.q = {}
+        self.full = {}
+
+    async def run(self, client):
+        while True:
+            try:
+                async with client.subscribe(
+                    topic=self.topic,
+                    subscription_name=self.subscription,
+                    consumer_name=self.consumer_name,
+                    schema=self.schema,
+                ) as consumer:
+                    while True:
+                        msg = await consumer.receive()
+
+                        # Acknowledge successful reception of the message
+                        await consumer.acknowledge(msg)
+
+                        try:
+                            id = msg.properties()["id"]
+                        except:
+                            id = None
+
+                        value = msg.value()
+                        if id in self.q:
+                            await self.q[id].put(value)
+
+                        for q in self.full.values():
+                            await q.put(value)
+
+            except Exception as e:
+                print("Exception:", e, flush=True)
+         
+            # If handler drops out, sleep a retry
+            await asyncio.sleep(2)
+
+    async def subscribe(self, id):
+        q = asyncio.Queue()
+        self.q[id] = q
+        return q
+
+    async def unsubscribe(self, id):
+        if id in self.q:
+            del self.q[id]
+    
+    async def subscribe_all(self, id):
+        q = asyncio.Queue()
+        self.full[id] = q
+        return q
+
+    async def unsubscribe_all(self, id):
+        if id in self.full:
+            del self.full[id]
+

{trustgraph-flow-0.17.1 → trustgraph-flow-0.17.3}/trustgraph/api/gateway/triples_load.py

@@ -27,10 +27,10 @@ class TriplesLoadEndpoint(SocketEndpoint):
             schema=JsonSchema(Triples)
         )
 
-    async def start(self):
+    async def start(self, client):
 
         self.task = asyncio.create_task(
-            self.publisher.run()
+            self.publisher.run(client)
         )
 
     async def listener(self, ws, running):

{trustgraph-flow-0.17.1 → trustgraph-flow-0.17.3}/trustgraph/api/gateway/triples_stream.py

@@ -26,10 +26,10 @@ class TriplesStreamEndpoint(SocketEndpoint):
             schema=JsonSchema(Triples)
         )
 
-    async def start(self):
+    async def start(self, client):
 
         self.task = asyncio.create_task(
-            self.subscriber.run()
+            self.subscriber.run(client)
         )
 
     async def async_thread(self, ws, running):

trustgraph-flow-0.17.3/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "0.17.3"

{trustgraph-flow-0.17.1 → trustgraph-flow-0.17.3}/trustgraph/query/doc_embeddings/pinecone/service.py

@@ -131,9 +131,14 @@ class Processor(ConsumerProducer):
         )
 
         parser.add_argument(
-            '-t', '--store-uri',
-            default=default_store_uri,
-            help=f'Milvus store URI (default: {default_store_uri})'
+            '-a', '--api-key',
+            default=default_api_key,
+            help='Pinecone API key. (default from PINECONE_API_KEY)'
+        )
+
+        parser.add_argument(
+            '-u', '--url',
+            help='Pinecone URL.  If unspecified, serverless is used'
         )
 
 def run():

{trustgraph-flow-0.17.1 → trustgraph-flow-0.17.3}/trustgraph_flow.egg-info/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.17.1
+Version: 0.17.3
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 License: UNKNOWN
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.17.1.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.17.3.tar.gz
 Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)

trustgraph-flow-0.17.1/trustgraph/api/gateway/publisher.py

@@ -1,41 +0,0 @@
-
-import asyncio
-import aiopulsar
-
-class Publisher:
-
-    def __init__(self, pulsar_host, topic, schema=None, max_size=10,
-                 chunking_enabled=False):
-        self.pulsar_host = pulsar_host
-        self.topic = topic
-        self.schema = schema
-        self.q = asyncio.Queue(maxsize=max_size)
-        self.chunking_enabled = chunking_enabled
-
-    async def run(self):
-
-        while True:
-
-            try:
-                async with aiopulsar.connect(self.pulsar_host) as client:
-                    async with client.create_producer(
-                            topic=self.topic,
-                            schema=self.schema,
-                            chunking_enabled=self.chunking_enabled,
-                    ) as producer:
-                        while True:
-                            id, item = await self.q.get()
-
-                            if id:
-                                await producer.send(item, { "id": id })
-                            else:
-                                await producer.send(item)
-
-            except Exception as e:
-                print("Exception:", e, flush=True)
-
-            # If handler drops out, sleep a retry
-            await asyncio.sleep(2)
-
-    async def send(self, id, msg):
-        await self.q.put((id, msg))

trustgraph-flow-0.17.1/trustgraph/api/gateway/subscriber.py

@@ -1,68 +0,0 @@
-
-import asyncio
-import aiopulsar
-
-class Subscriber:
-
-    def __init__(self, pulsar_host, topic, subscription, consumer_name,
-                 schema=None, max_size=10):
-        self.pulsar_host = pulsar_host
-        self.topic = topic
-        self.subscription = subscription
-        self.consumer_name = consumer_name
-        self.schema = schema
-        self.q = {}
-        self.full = {}
-
-    async def run(self):
-        while True:
-            try:
-                async with aiopulsar.connect(self.pulsar_host) as client:
-                    async with client.subscribe(
-                        topic=self.topic,
-                        subscription_name=self.subscription,
-                        consumer_name=self.consumer_name,
-                        schema=self.schema,
-                    ) as consumer:
-                        while True:
-                            msg = await consumer.receive()
-
-                            # Acknowledge successful reception of the message
-                            await consumer.acknowledge(msg)
-
-                            try:
-                                id = msg.properties()["id"]
-                            except:
-                                id = None
-
-                            value = msg.value()
-                            if id in self.q:
-                                await self.q[id].put(value)
-
-                            for q in self.full.values():
-                                await q.put(value)
-
-            except Exception as e:
-                print("Exception:", e, flush=True)
-         
-            # If handler drops out, sleep a retry
-            await asyncio.sleep(2)
-
-    async def subscribe(self, id):
-        q = asyncio.Queue()
-        self.q[id] = q
-        return q
-
-    async def unsubscribe(self, id):
-        if id in self.q:
-            del self.q[id]
-    
-    async def subscribe_all(self, id):
-        q = asyncio.Queue()
-        self.full[id] = q
-        return q
-
-    async def unsubscribe_all(self, id):
-        if id in self.full:
-            del self.full[id]
-

trustgraph-flow-0.17.1/trustgraph/flow_version.py

@@ -1 +0,0 @@
-__version__ = "0.17.1"