trustgraph-flow 0.16.0__tar.gz → 0.16.2__tar.gz

{trustgraph-flow-0.16.0 → trustgraph-flow-0.16.2}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.16.0
+Version: 0.16.2
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 License: UNKNOWN
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.16.0.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.16.2.tar.gz
 Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)

trustgraph-flow-0.16.2/scripts/api-gateway

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.api.gateway import run
+
+run()
+

trustgraph-flow-0.16.0/scripts/api-gateway → trustgraph-flow-0.16.2/trustgraph/api/gateway/service.py

@@ -1,4 +1,10 @@
-#!/usr/bin/env python3
+
+"""
+API gateway.  Offers HTTP services which are translated to interaction on the
+Pulsar bus.
+"""
+
+module = ".".join(__name__.split(".")[1:-1])
 
 # FIXME: Subscribes to Pulsar unnecessarily, should only do it when there
 # are active listeners
@@ -7,72 +13,111 @@
 # to fail and are not failed or retried
 
 import asyncio
+import argparse
 from aiohttp import web
 import json
 import logging
 import uuid
 import os
+import base64
 
 import pulsar
 from pulsar.asyncio import Client
 from pulsar.schema import JsonSchema
 import _pulsar
 import aiopulsar
+from prometheus_client import start_http_server
+
+from ... log_level import LogLevel
 
 from trustgraph.clients.llm_client import LlmClient
 from trustgraph.clients.prompt_client import PromptClient
 
-from trustgraph.schema import TextCompletionRequest, TextCompletionResponse
-from trustgraph.schema import text_completion_request_queue
-from trustgraph.schema import text_completion_response_queue
+from ... schema import Value, Metadata, Document, TextDocument, Triple
+
+from ... schema import TextCompletionRequest, TextCompletionResponse
+from ... schema import text_completion_request_queue
+from ... schema import text_completion_response_queue
+
+from ... schema import PromptRequest, PromptResponse
+from ... schema import prompt_request_queue
+from ... schema import prompt_response_queue
 
-from trustgraph.schema import PromptRequest, PromptResponse
-from trustgraph.schema import prompt_request_queue
-from trustgraph.schema import prompt_response_queue
+from ... schema import GraphRagQuery, GraphRagResponse
+from ... schema import graph_rag_request_queue
+from ... schema import graph_rag_response_queue
 
-from trustgraph.schema import GraphRagQuery, GraphRagResponse
-from trustgraph.schema import graph_rag_request_queue
-from trustgraph.schema import graph_rag_response_queue
+from ... schema import TriplesQueryRequest, TriplesQueryResponse
+from ... schema import triples_request_queue
+from ... schema import triples_response_queue
 
-from trustgraph.schema import TriplesQueryRequest, TriplesQueryResponse, Value
-from trustgraph.schema import triples_request_queue
-from trustgraph.schema import triples_response_queue
+from ... schema import AgentRequest, AgentResponse
+from ... schema import agent_request_queue
+from ... schema import agent_response_queue
 
-from trustgraph.schema import AgentRequest, AgentResponse
-from trustgraph.schema import agent_request_queue
-from trustgraph.schema import agent_response_queue
+from ... schema import EmbeddingsRequest, EmbeddingsResponse
+from ... schema import embeddings_request_queue
+from ... schema import embeddings_response_queue
 
-from trustgraph.schema import EmbeddingsRequest, EmbeddingsResponse
-from trustgraph.schema import embeddings_request_queue
-from trustgraph.schema import embeddings_response_queue
+from ... schema import document_ingest_queue, text_ingest_queue
 
 logger = logging.getLogger("api")
 logger.setLevel(logging.INFO)
 
-pulsar_host = os.getenv("PULSAR_HOST", "pulsar://pulsar:6650")
-TIME_OUT = 600
+default_pulsar_host = os.getenv("PULSAR_HOST", "pulsar://pulsar:6650")
+default_timeout = 600
+default_port = 8088
+
+def to_value(x):
+    if x.startswith("http:") or x.startswith("https:"):
+        return Value(value=x, is_uri=True)
+    else:
+        return Value(value=x, is_uri=True)
+
+def to_subgraph(x):
+    return [
+        Triple(
+            s=to_value(t["s"]),
+            p=to_value(t["p"]),
+            o=to_value(t["o"])
+        )
+        for t in x
+    ]
 
 class Publisher:
 
-    def __init__(self, pulsar_host, topic, schema=None, max_size=10):
+    def __init__(self, pulsar_host, topic, schema=None, max_size=10,
+                 chunking_enabled=False):
         self.pulsar_host = pulsar_host
         self.topic = topic
         self.schema = schema
         self.q = asyncio.Queue(maxsize=max_size)
+        self.chunking_enabled = chunking_enabled
 
     async def run(self):
-        try:
-            async with aiopulsar.connect(self.pulsar_host) as client:
-                async with client.create_producer(
-                        topic=self.topic,
-                        schema=self.schema,
-                ) as producer:
-                    while True:
-                        id, item = await self.q.get()
-                        await producer.send(item, { "id": id })
-    #                     print("message out")
-        except Exception as e:
-            print("Exception:", e, flush=True)
+
+        while True:
+
+            try:
+                async with aiopulsar.connect(self.pulsar_host) as client:
+                    async with client.create_producer(
+                            topic=self.topic,
+                            schema=self.schema,
+                            chunking_enabled=self.chunking_enabled,
+                    ) as producer:
+                        while True:
+                            id, item = await self.q.get()
+
+                            if id:
+                                await producer.send(item, { "id": id })
+                            else:
+                                await producer.send(item)
+
+            except Exception as e:
+                print("Exception:", e, flush=True)
+
+            # If handler drops out, sleep a retry
+            await asyncio.sleep(2)
 
     async def send(self, id, msg):
         await self.q.put((id, msg))
@@ -89,24 +134,27 @@ class Subscriber:
         self.q = {}
 
     async def run(self):
-        try:
-            async with aiopulsar.connect(pulsar_host) as client:
-                async with client.subscribe(
-                    topic=self.topic,
-                    subscription_name=self.subscription,
-                    consumer_name=self.consumer_name,
-                    schema=self.schema,
-                ) as consumer:
-                    while True:
-                        msg = await consumer.receive()
-    #                     print("message in", self.topic)
-                        id = msg.properties()["id"]
-                        value = msg.value()
-                        if id in self.q:
-                            await self.q[id].put(value)
-        except Exception as e:
-            print("Exception:", e, flush=True)
+        while True:
+            try:
+                async with aiopulsar.connect(self.pulsar_host) as client:
+                    async with client.subscribe(
+                        topic=self.topic,
+                        subscription_name=self.subscription,
+                        consumer_name=self.consumer_name,
+                        schema=self.schema,
+                    ) as consumer:
+                        while True:
+                            msg = await consumer.receive()
+                            id = msg.properties()["id"]
+                            value = msg.value()
+                            if id in self.q:
+                                await self.q[id].put(value)
+            except Exception as e:
+                print("Exception:", e, flush=True)
          
+            # If handler drops out, sleep a retry
+            await asyncio.sleep(2)
+
     async def subscribe(self, id):
         q = asyncio.Queue()
         self.q[id] = q
@@ -120,75 +168,93 @@ class Api:
 
     def __init__(self, **config):
 
-        self.port = int(config.get("port", "8088"))
-        self.app = web.Application(middlewares=[])
+        self.app = web.Application(
+            middlewares=[],
+            client_max_size=256 * 1024 * 1024
+        )
+
+        self.port = int(config.get("port", default_port))
+        self.timeout = int(config.get("timeout", default_timeout))
+        self.pulsar_host = config.get("pulsar_host", default_pulsar_host)
 
         self.llm_out = Publisher(
-            pulsar_host, text_completion_request_queue,
+            self.pulsar_host, text_completion_request_queue,
             schema=JsonSchema(TextCompletionRequest)
         )
 
         self.llm_in = Subscriber(
-            pulsar_host, text_completion_response_queue,
+            self.pulsar_host, text_completion_response_queue,
             "api-gateway", "api-gateway",
             JsonSchema(TextCompletionResponse)
         )
 
         self.prompt_out = Publisher(
-            pulsar_host, prompt_request_queue,
+            self.pulsar_host, prompt_request_queue,
             schema=JsonSchema(PromptRequest)
         )
 
         self.prompt_in = Subscriber(
-            pulsar_host, prompt_response_queue,
+            self.pulsar_host, prompt_response_queue,
             "api-gateway", "api-gateway",
             JsonSchema(PromptResponse)
         )
 
         self.graph_rag_out = Publisher(
-            pulsar_host, graph_rag_request_queue,
+            self.pulsar_host, graph_rag_request_queue,
             schema=JsonSchema(GraphRagQuery)
         )
 
         self.graph_rag_in = Subscriber(
-            pulsar_host, graph_rag_response_queue,
+            self.pulsar_host, graph_rag_response_queue,
             "api-gateway", "api-gateway",
             JsonSchema(GraphRagResponse)
         )
 
         self.triples_query_out = Publisher(
-            pulsar_host, triples_request_queue,
+            self.pulsar_host, triples_request_queue,
             schema=JsonSchema(TriplesQueryRequest)
         )
 
         self.triples_query_in = Subscriber(
-            pulsar_host, triples_response_queue,
+            self.pulsar_host, triples_response_queue,
             "api-gateway", "api-gateway",
             JsonSchema(TriplesQueryResponse)
         )
 
         self.agent_out = Publisher(
-            pulsar_host, agent_request_queue,
+            self.pulsar_host, agent_request_queue,
             schema=JsonSchema(AgentRequest)
         )
 
         self.agent_in = Subscriber(
-            pulsar_host, agent_response_queue,
+            self.pulsar_host, agent_response_queue,
             "api-gateway", "api-gateway",
             JsonSchema(AgentResponse)
         )
 
         self.embeddings_out = Publisher(
-            pulsar_host, embeddings_request_queue,
+            self.pulsar_host, embeddings_request_queue,
             schema=JsonSchema(EmbeddingsRequest)
         )
 
         self.embeddings_in = Subscriber(
-            pulsar_host, embeddings_response_queue,
+            self.pulsar_host, embeddings_response_queue,
             "api-gateway", "api-gateway",
             JsonSchema(EmbeddingsResponse)
         )
 
+        self.document_out = Publisher(
+            self.pulsar_host, document_ingest_queue,
+            schema=JsonSchema(Document),
+            chunking_enabled=True,
+        )
+
+        self.text_out = Publisher(
+            self.pulsar_host, text_ingest_queue,
+            schema=JsonSchema(TextDocument),
+            chunking_enabled=True,
+        )
+
         self.app.add_routes([
             web.post("/api/v1/text-completion", self.llm),
             web.post("/api/v1/prompt", self.prompt),
@@ -196,6 +262,8 @@ class Api:
             web.post("/api/v1/triples-query", self.triples_query),
             web.post("/api/v1/agent", self.agent),
             web.post("/api/v1/embeddings", self.embeddings),
+            web.post("/api/v1/load/document", self.load_document),
+            web.post("/api/v1/load/text", self.load_text),
         ])
 
     async def llm(self, request):
@@ -217,7 +285,7 @@ class Api:
             )
 
             try:
-                resp = await asyncio.wait_for(q.get(), TIME_OUT)
+                resp = await asyncio.wait_for(q.get(), self.timeout)
             except:
                 raise RuntimeError("Timeout waiting for response")
 
@@ -264,7 +332,7 @@ class Api:
             )
 
             try:
-                resp = await asyncio.wait_for(q.get(), TIME_OUT)
+                resp = await asyncio.wait_for(q.get(), self.timeout)
             except:
                 raise RuntimeError("Timeout waiting for response")
 
@@ -312,7 +380,7 @@ class Api:
             )
 
             try:
-                resp = await asyncio.wait_for(q.get(), TIME_OUT)
+                resp = await asyncio.wait_for(q.get(), self.timeout)
             except:
                 raise RuntimeError("Timeout waiting for response")
 
@@ -346,26 +414,17 @@ class Api:
             q = await self.triples_query_in.subscribe(id)
 
             if "s" in data:
-                if data["s"].startswith("http:") or data["s"].startswith("https:"):
-                    s = Value(value=data["s"], is_uri=True)
-                else:
-                    s = Value(value=data["s"], is_uri=True)
+                s = to_value(data["s"])
             else:
                 s = None
 
             if "p" in data:
-                if data["p"].startswith("http:") or data["p"].startswith("https:"):
-                    p = Value(value=data["p"], is_uri=True)
-                else:
-                    p = Value(value=data["p"], is_uri=True)
+                p = to_value(data["p"])
             else:
                 p = None
 
             if "o" in data:
-                if data["o"].startswith("http:") or data["o"].startswith("https:"):
-                    o = Value(value=data["o"], is_uri=True)
-                else:
-                    o = Value(value=data["o"], is_uri=True)
+                o = to_value(data["o"])
             else:
                 o = None
 
@@ -382,7 +441,7 @@ class Api:
             )
 
             try:
-                resp = await asyncio.wait_for(q.get(), TIME_OUT)
+                resp = await asyncio.wait_for(q.get(), self.timeout)
             except:
                 raise RuntimeError("Timeout waiting for response")
 
@@ -442,7 +501,7 @@ class Api:
 
             while True:
                 try:
-                    resp = await asyncio.wait_for(q.get(), TIME_OUT)
+                    resp = await asyncio.wait_for(q.get(), self.timeout)
                 except:
                     raise RuntimeError("Timeout waiting for response")
 
@@ -492,7 +551,7 @@ class Api:
             )
 
             try:
-                resp = await asyncio.wait_for(q.get(), TIME_OUT)
+                resp = await asyncio.wait_for(q.get(), self.timeout)
             except:
                 raise RuntimeError("Timeout waiting for response")
 
@@ -515,6 +574,92 @@ class Api:
         finally:
             await self.embeddings_in.unsubscribe(id)
 
+    async def load_document(self, request):
+
+        try:
+
+            data = await request.json()
+
+            if "metadata" in data:
+                metadata = to_subgraph(data["metadata"])
+            else:
+                metadata = []
+
+            # Doing a base64 decode/encode here to make sure the
+            # content is valid base64
+            doc = base64.b64decode(data["data"])
+
+            resp = await self.document_out.send(
+                None,
+                Document(
+                    metadata=Metadata(
+                        id=data.get("id"),
+                        metadata=metadata,
+                        user=data.get("user", "trustgraph"),
+                        collection=data.get("collection", "default"),
+                    ),
+                    data=base64.b64encode(doc).decode("utf-8")
+                )
+            )
+
+            print("Document loaded.")
+
+            return web.json_response(
+                { }
+            )
+
+        except Exception as e:
+            logging.error(f"Exception: {e}")
+
+            return web.json_response(
+                { "error": str(e) }
+            )
+
+    async def load_text(self, request):
+
+        try:
+
+            data = await request.json()
+
+            if "metadata" in data:
+                metadata = to_subgraph(data["metadata"])
+            else:
+                metadata = []
+
+            if "charset" in data:
+                charset = data["charset"]
+            else:
+                charset = "utf-8"
+
+            # Text is base64 encoded
+            text = base64.b64decode(data["text"]).decode(charset)
+
+            resp = await self.text_out.send(
+                None,
+                TextDocument(
+                    metadata=Metadata(
+                        id=data.get("id"),
+                        metadata=metadata,
+                        user=data.get("user", "trustgraph"),
+                        collection=data.get("collection", "default"),
+                    ),
+                    text=text,
+                )
+            )
+
+            print("Text document loaded.")
+
+            return web.json_response(
+                { }
+            )
+
+        except Exception as e:
+            logging.error(f"Exception: {e}")
+
+            return web.json_response(
+                { "error": str(e) }
+            )
+
     async def app_factory(self):
 
         self.llm_pub_task = asyncio.create_task(self.llm_in.run())
@@ -543,11 +688,71 @@ class Api:
             self.embeddings_out.run()
         )
 
+        self.doc_ingest_pub_task = asyncio.create_task(self.document_out.run())
+
+        self.text_ingest_pub_task = asyncio.create_task(self.text_out.run())
+
         return self.app
 
     def run(self):
         web.run_app(self.app_factory(), port=self.port)
 
-a = Api()
-a.run()
+def run():
+
+
+    parser = argparse.ArgumentParser(
+        prog="api-gateway",
+        description=__doc__
+    )
+
+    parser.add_argument(
+        '-p', '--pulsar-host',
+        default=default_pulsar_host,
+        help=f'Pulsar host (default: {default_pulsar_host})',
+    )
+
+    parser.add_argument(
+        '--port',
+        type=int,
+        default=default_port,
+        help=f'Port number to listen on (default: {default_port})',
+    )
+
+    parser.add_argument(
+        '--timeout',
+        type=int,
+        default=default_timeout,
+        help=f'API request timeout in seconds (default: {default_timeout})',
+    )
+
+    parser.add_argument(
+        '-l', '--log-level',
+        type=LogLevel,
+        default=LogLevel.INFO,
+        choices=list(LogLevel),
+        help=f'Output queue (default: info)'
+    )
+
+    parser.add_argument(
+        '--metrics',
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help=f'Metrics enabled (default: true)',
+    )
+
+    parser.add_argument(
+        '-P', '--metrics-port',
+        type=int,
+        default=8000,
+        help=f'Prometheus metrics port (default: 8000)',
+    )
+
+    args = parser.parse_args()
+    args = vars(args)
+
+    if args["metrics"]:
+        start_http_server(args["metrics_port"])
+
+    a = Api(**args)
+    a.run()
 

trustgraph-flow-0.16.2/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "0.16.2"

trustgraph-flow-0.16.2/trustgraph/model/prompt/template/__main__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+from . service import run
+
+if __name__ == '__main__':
+    run()
+

trustgraph-flow-0.16.2/trustgraph/query/triples/neo4j/__init__.py

@@ -0,0 +1,3 @@
+
+from . service import *
+

{trustgraph-flow-0.16.0 → trustgraph-flow-0.16.2}/trustgraph_flow.egg-info/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.16.0
+Version: 0.16.2
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 License: UNKNOWN
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.16.0.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.16.2.tar.gz
 Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)

{trustgraph-flow-0.16.0 → trustgraph-flow-0.16.2}/trustgraph_flow.egg-info/SOURCES.txt

@@ -50,6 +50,10 @@ trustgraph/agent/react/agent_manager.py
 trustgraph/agent/react/service.py
 trustgraph/agent/react/tools.py
 trustgraph/agent/react/types.py
+trustgraph/api/__init__.py
+trustgraph/api/gateway/__init__.py
+trustgraph/api/gateway/__main__.py
+trustgraph/api/gateway/service.py
 trustgraph/chunking/__init__.py
 trustgraph/chunking/recursive/__init__.py
 trustgraph/chunking/recursive/__main__.py

trustgraph-flow-0.16.0/trustgraph/flow_version.py

@@ -1 +0,0 @@
-__version__ = "0.16.0"