trustgraph-flow 0.18.7__tar.gz → 0.18.9__tar.gz

{trustgraph-flow-0.18.7 → trustgraph-flow-0.18.9}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.18.7
+Version: 0.18.9
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 License: UNKNOWN
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.18.7.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.18.9.tar.gz
 Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)

trustgraph-flow-0.18.9/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "0.18.9"

trustgraph-flow-0.18.9/trustgraph/gateway/document_load.py

@@ -0,0 +1,42 @@
+
+import base64
+
+from .. schema import Document
+from .. schema import document_ingest_queue
+
+from . sender import ServiceSender
+from . serialize import to_subgraph
+
+class DocumentLoadSender(ServiceSender):
+    def __init__(self, pulsar_host):
+
+        super(DocumentLoadSender, self).__init__(
+            pulsar_host=pulsar_host,
+            request_queue=document_ingest_queue,
+            request_schema=Document,
+        )
+
+    def to_request(self, body):
+
+        if "metadata" in data:
+            metadata = to_subgraph(data["metadata"])
+        else:
+            metadata = []
+
+        # Doing a base64 decoe/encode here to make sure the
+        # content is valid base64
+        doc = base64.b64decode(data["data"])
+
+        print("Document received")
+
+        return Document(
+            metadata=Metadata(
+                id=data.get("id"),
+                metadata=metadata,
+                user=data.get("user", "trustgraph"),
+                collection=data.get("collection", "default"),
+            ),
+            data=base64.b64encode(doc).decode("utf-8")
+        )
+
+

{trustgraph-flow-0.18.7 → trustgraph-flow-0.18.9}/trustgraph/gateway/embeddings.py

@@ -26,4 +26,3 @@ class EmbeddingsRequestor(ServiceRequestor):
     def from_response(self, message):
         return { "vectors": message.vectors }, True
 
-

trustgraph-flow-0.18.9/trustgraph/gateway/mux.py

@@ -0,0 +1,169 @@
+
+import asyncio
+import queue
+from pulsar.schema import JsonSchema
+import uuid
+from aiohttp import web, WSMsgType
+
+from . socket import SocketEndpoint
+from . text_completion import TextCompletionRequestor
+
+MAX_OUTSTANDING_REQUESTS = 15
+WORKER_CLOSE_WAIT = 0.01
+START_REQUEST_WAIT = 0.1
+
+# This buffers requests until task start, so short-lived
+MAX_QUEUE_SIZE = 10
+
+class MuxEndpoint(SocketEndpoint):
+
+    def __init__(
+            self, pulsar_host, auth,
+            services,
+            path="/api/v1/socket",
+    ):
+
+        super(MuxEndpoint, self).__init__(
+            endpoint_path=path, auth=auth,
+        )
+
+        self.services = services
+
+    async def start(self):
+        pass
+
+    async def maybe_tidy_workers(self, workers):
+
+        while True:
+
+            try:
+
+                await asyncio.wait_for(
+                    asyncio.shield(workers[0]),
+                    WORKER_CLOSE_WAIT
+                )
+
+                # worker[0] now stopped
+                # FIXME: Delete reference???
+
+                workers.pop(0)
+
+                if len(workers) == 0:
+                    break
+
+                # Loop iterates to try the next worker
+
+            except TimeoutError:
+                # worker[0] still running, move on
+                break
+
+    async def start_request_task(self, ws, id, svc, request, workers):
+
+        if svc not in self.services:
+            await ws.send_json({"id": id, "error": "Service not recognised"})
+            return
+            
+        requestor = self.services[svc]
+
+        async def responder(resp, fin):
+            await ws.send_json({
+                "id": id,
+                "response": resp,
+                "complete": fin,
+            })
+
+        # Wait for outstanding requests to go below MAX_OUTSTANDING_REQUESTS
+        while len(workers) > MAX_OUTSTANDING_REQUESTS:
+
+            # Fixes deadlock
+            # FIXME: Put it in its own loop
+            await asyncio.sleep(START_REQUEST_WAIT)
+
+            await self.maybe_tidy_workers(workers)
+
+        worker = asyncio.create_task(
+            requestor.process(request, responder)
+        )
+
+        workers.append(worker)
+
+    async def async_thread(self, ws, running, q):
+
+        # Worker threads, servicing
+        workers = []
+
+        while running.get():
+
+            try:
+
+                if len(workers) > 0:
+                    await self.maybe_tidy_workers(workers)
+
+                # Get next request on queue
+                id, svc, request = await asyncio.wait_for(q.get(), 1)
+
+            except TimeoutError:
+                continue
+
+            except Exception as e:
+                # This is an internal working error, may not be recoverable
+                print("Exception:", e)
+                await ws.send_json({"id": id, "error": str(e)})
+                break
+
+            try:
+                print(id, svc, request)
+                await self.start_request_task(ws, id, svc, request, workers)
+
+            except Exception as e:
+                print("Exception2:", e)
+                await ws.send_json({"error": str(e)})
+
+        running.stop()
+
+    async def listener(self, ws, running):
+
+        # The outstanding request queue, max size is MAX_QUEUE_SIZE
+        q = asyncio.Queue(maxsize=MAX_QUEUE_SIZE)
+
+        async_task = asyncio.create_task(self.async_thread(
+            ws, running, q
+        ))
+        
+        async for msg in ws:
+
+            # On error, finish
+            if msg.type == WSMsgType.TEXT:
+
+                try:
+
+                    data = msg.json()
+
+                    if data["service"] not in self.services:
+                        raise RuntimeError("Bad service")
+
+                    if "request" not in data:
+                        raise RuntimeError("Bad message")
+
+                    if "id" not in data:
+                        raise RuntimeError("Bad message")
+
+                    await q.put(
+                        (data["id"], data["service"], data["request"])
+                    )
+
+                except Exception as e:
+
+                    await ws.send_json({"error": str(e)})
+                    continue
+
+            elif msg.type == WSMsgType.ERROR:
+                break
+            elif msg.type == WSMsgType.CLOSE:
+                break
+            else:
+                break
+
+        running.stop()
+
+        await async_task

trustgraph-flow-0.18.9/trustgraph/gateway/sender.py

@@ -0,0 +1,49 @@
+
+# Like ServiceRequestor, but just fire-and-forget instead of request/response
+
+import asyncio
+from pulsar.schema import JsonSchema
+import uuid
+import logging
+
+from . publisher import Publisher
+
+logger = logging.getLogger("sender")
+logger.setLevel(logging.INFO)
+
+class ServiceSender:
+
+    def __init__(
+            self,
+            pulsar_host,
+            request_queue, request_schema,
+    ):
+
+        self.pub = Publisher(
+            pulsar_host, request_queue,
+            schema=JsonSchema(request_schema)
+        )
+
+    async def start(self):
+
+        self.pub.start()
+
+    def to_request(self, request):
+        raise RuntimeError("Not defined")
+
+    async def process(self, request, responder=None):
+
+        try:
+
+            await asyncio.to_thread(
+                self.pub.send, None, self.to_request(request)
+            )
+
+            return {}
+
+        except Exception as e:
+
+            logging.error(f"Exception: {e}")
+
+            return { "error": str(e) }
+

{trustgraph-flow-0.18.7 → trustgraph-flow-0.18.9}/trustgraph/gateway/service.py

@@ -24,9 +24,6 @@ from prometheus_client import start_http_server
 
 from .. log_level import LogLevel
 
-from .. schema import Metadata, Document, TextDocument
-from .. schema import document_ingest_queue, text_ingest_queue
-
 from . serialize import to_subgraph
 from . running import Running
 from . publisher import Publisher
@@ -46,6 +43,8 @@ from . graph_embeddings_stream import GraphEmbeddingsStreamEndpoint
 from . triples_load import TriplesLoadEndpoint
 from . graph_embeddings_load import GraphEmbeddingsLoadEndpoint
 from . mux import MuxEndpoint
+from . document_load import DocumentLoadSender
+from . text_load import TextLoadSender
 
 from . endpoint import ServiceEndpoint
 from . auth import Authenticator
@@ -120,6 +119,12 @@ class Api:
                 pulsar_host=self.pulsar_host, timeout=self.timeout,
                 auth = self.auth,
             ),
+            "document-load": DocumentLoadSender(
+                pulsar_host=self.pulsar_host,
+            ),
+            "text-load": TextLoadSender(
+                pulsar_host=self.pulsar_host,
+            ),
         }
 
         self.endpoints = [
@@ -164,6 +169,14 @@ class Api:
                 endpoint_path = "/api/v1/internet-search", auth=self.auth,
                 requestor = self.services["internet-search"],
             ),
+            ServiceEndpoint(
+                endpoint_path = "/api/v1/load/document", auth=self.auth,
+                requestor = self.services["document-load"],
+            ),
+            ServiceEndpoint(
+                endpoint_path = "/api/v1/load/text", auth=self.auth,
+                requestor = self.services["text-load"],
+            ),
             TriplesStreamEndpoint(
                 pulsar_host=self.pulsar_host,
                 auth = self.auth,
@@ -187,122 +200,14 @@ class Api:
             ),
         ]
 
-        self.document_out = Publisher(
-            self.pulsar_host, document_ingest_queue,
-            schema=JsonSchema(Document),
-            chunking_enabled=True,
-        )
-
-        self.text_out = Publisher(
-            self.pulsar_host, text_ingest_queue,
-            schema=JsonSchema(TextDocument),
-            chunking_enabled=True,
-        )
-
         for ep in self.endpoints:
             ep.add_routes(self.app)
 
-        self.app.add_routes([
-            web.post("/api/v1/load/document", self.load_document),
-            web.post("/api/v1/load/text", self.load_text),
-        ])
-
-    async def load_document(self, request):
-
-        try:
-
-            data = await request.json()
-
-            if "metadata" in data:
-                metadata = to_subgraph(data["metadata"])
-            else:
-                metadata = []
-
-            # Doing a base64 decoe/encode here to make sure the
-            # content is valid base64
-            doc = base64.b64decode(data["data"])
-
-            resp = await asyncio.to_thread(
-                self.document_out.send,
-                None,
-                Document(
-                    metadata=Metadata(
-                        id=data.get("id"),
-                        metadata=metadata,
-                        user=data.get("user", "trustgraph"),
-                        collection=data.get("collection", "default"),
-                    ),
-                    data=base64.b64encode(doc).decode("utf-8")
-                )
-            )
-
-            print("Document loaded.")
-
-            return web.json_response(
-                { }
-            )
-
-        except Exception as e:
-            logging.error(f"Exception: {e}")
-
-            return web.json_response(
-                { "error": str(e) }
-            )
-
-    async def load_text(self, request):
-
-        try:
-
-            data = await request.json()
-
-            if "metadata" in data:
-                metadata = to_subgraph(data["metadata"])
-            else:
-                metadata = []
-
-            if "charset" in data:
-                charset = data["charset"]
-            else:
-                charset = "utf-8"
-
-            # Text is base64 encoded
-            text = base64.b64decode(data["text"]).decode(charset)
-
-            resp = await asyncio.to_thread(
-                self.text_out.send,
-                None,
-                TextDocument(
-                    metadata=Metadata(
-                        id=data.get("id"),
-                        metadata=metadata,
-                        user=data.get("user", "trustgraph"),
-                        collection=data.get("collection", "default"),
-                    ),
-                    text=text,
-                )
-            )
-
-            print("Text document loaded.")
-
-            return web.json_response(
-                { }
-            )
-
-        except Exception as e:
-            logging.error(f"Exception: {e}")
-
-            return web.json_response(
-                { "error": str(e) }
-            )
-
     async def app_factory(self):
 
         for ep in self.endpoints:
             await ep.start()
 
-        self.document_out.start()
-        self.text_out.start()
-
         return self.app
 
     def run(self):

{trustgraph-flow-0.18.7 → trustgraph-flow-0.18.9}/trustgraph/gateway/socket.py

@@ -22,25 +22,16 @@ class SocketEndpoint:
         
         async for msg in ws:
             # On error, finish
-            if msg.type == WSMsgType.ERROR:
-                break
+            if msg.type == WSMsgType.TEXT:
+                # Ignore incoming message
+                continue
+            elif msg.type == WSMsgType.BINARY:
+                # Ignore incoming message
+                continue
             else:
-                # Ignore incoming messages
-                pass
+                break
 
         running.stop()
-
-    async def async_thread(self, ws, running):
-
-        while running.get():
-            try:
-                await asyncio.sleep(1)
-
-            except TimeoutError:
-                continue
-
-            except Exception as e:
-                print(f"Exception: {str(e)}", flush=True)
         
     async def handle(self, request):
 
@@ -56,12 +47,8 @@ class SocketEndpoint:
         ws = web.WebSocketResponse()
         await ws.prepare(request)
 
-        task = asyncio.create_task(self.async_thread(ws, running))
-
         try:
-
             await self.listener(ws, running)
-
         except Exception as e:
             print(e, flush=True)
 
@@ -69,8 +56,6 @@ class SocketEndpoint:
 
         await ws.close()
 
-        await task
-
         return ws
 
     async def start(self):

trustgraph-flow-0.18.9/trustgraph/gateway/text_load.py

@@ -0,0 +1,45 @@
+
+import base64
+
+from .. schema import TextDocument, Metadata
+from .. schema import text_ingest_queue
+
+from . sender import ServiceSender
+from . serialize import to_subgraph
+
+class TextLoadSender(ServiceSender):
+    def __init__(self, pulsar_host):
+
+        super(TextLoadSender, self).__init__(
+            pulsar_host=pulsar_host,
+            request_queue=text_ingest_queue,
+            request_schema=TextDocument,
+        )
+
+    def to_request(self, body):
+
+        if "metadata" in body:
+            metadata = to_subgraph(body["metadata"])
+        else:
+            metadata = []
+
+        if "charset" in body:
+            charset = body["charset"]
+        else:
+            charset = "utf-8"
+
+        # Text is base64 encoded
+        text = base64.b64decode(body["text"]).decode(charset)
+
+        print("Text document received")
+
+        return TextDocument(
+            metadata=Metadata(
+                id=body.get("id"),
+                metabody=metadata,
+                user=body.get("user", "trustgraph"),
+                collection=body.get("collection", "default"),
+            ),
+            text=text,
+        )
+

{trustgraph-flow-0.18.7 → trustgraph-flow-0.18.9}/trustgraph_flow.egg-info/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.18.7
+Version: 0.18.9
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 License: UNKNOWN
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.18.7.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.18.9.tar.gz
 Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)

{trustgraph-flow-0.18.7 → trustgraph-flow-0.18.9}/trustgraph_flow.egg-info/SOURCES.txt

@@ -106,6 +106,7 @@ trustgraph/gateway/__main__.py
 trustgraph/gateway/agent.py
 trustgraph/gateway/auth.py
 trustgraph/gateway/dbpedia.py
+trustgraph/gateway/document_load.py
 trustgraph/gateway/embeddings.py
 trustgraph/gateway/encyclopedia.py
 trustgraph/gateway/endpoint.py
@@ -119,11 +120,13 @@ trustgraph/gateway/prompt.py
 trustgraph/gateway/publisher.py
 trustgraph/gateway/requestor.py
 trustgraph/gateway/running.py
+trustgraph/gateway/sender.py
 trustgraph/gateway/serialize.py
 trustgraph/gateway/service.py
 trustgraph/gateway/socket.py
 trustgraph/gateway/subscriber.py
 trustgraph/gateway/text_completion.py
+trustgraph/gateway/text_load.py
 trustgraph/gateway/triples_load.py
 trustgraph/gateway/triples_query.py
 trustgraph/gateway/triples_stream.py

trustgraph-flow-0.18.7/trustgraph/flow_version.py

@@ -1 +0,0 @@
-__version__ = "0.18.7"

trustgraph-flow-0.18.7/trustgraph/gateway/mux.py

@@ -1,94 +0,0 @@
-
-import asyncio
-import queue
-from pulsar.schema import JsonSchema
-import uuid
-from aiohttp import web, WSMsgType
-
-from . socket import SocketEndpoint
-from . text_completion import TextCompletionRequestor
-
-class MuxEndpoint(SocketEndpoint):
-
-    def __init__(
-            self, pulsar_host, auth,
-            services,
-            path="/api/v1/socket",
-    ):
-
-        super(MuxEndpoint, self).__init__(
-            endpoint_path=path, auth=auth,
-        )
-
-        self.q = asyncio.Queue(maxsize=10)
-
-        self.services = services
-
-    async def start(self):
-        pass
-
-    async def async_thread(self, ws, running):
-
-        while running.get():
-
-            try:
-                id, svc, request = await asyncio.wait_for(self.q.get(), 1)
-            except TimeoutError:
-                continue
-            except Exception as e:
-                await ws.send_json({"id": id, "error": str(e)})
-
-            try:
-
-                print(svc, request)
-
-                requestor = self.services[svc]
-
-                async def responder(resp, fin):
-                    await ws.send_json({
-                        "id": id,
-                        "response": resp,
-                        "complete": fin,
-                    })
-
-                resp = await requestor.process(request, responder)
-
-            except Exception as e:
-
-                await ws.send_json({"error": str(e)})
-
-        running.stop()
-
-    async def listener(self, ws, running):
-        
-        async for msg in ws:
-
-            # On error, finish
-            if msg.type == WSMsgType.ERROR:
-                break
-            else:
-
-                try:
-
-                    data = msg.json()
-
-                    if data["service"] not in self.services:
-                        raise RuntimeError("Bad service")
-
-                    if "request" not in data:
-                        raise RuntimeError("Bad message")
-
-                    if "id" not in data:
-                        raise RuntimeError("Bad message")
-
-                    await self.q.put(
-                        (data["id"], data["service"], data["request"])
-                    )
-
-                except Exception as e:
-
-                    await ws.send_json({"error": str(e)})
-                    continue
-
-        running.stop()
-