trustgraph-flow 1.0.15__tar.gz → 1.0.17__tar.gz

{trustgraph-flow-1.0.15 → trustgraph-flow-1.0.17}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 1.0.15
+Version: 1.0.17
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v1.0.15.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v1.0.17.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

trustgraph-flow-1.0.17/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "1.0.17"

{trustgraph-flow-1.0.15 → trustgraph-flow-1.0.17}/trustgraph/gateway/dispatch/agent.py

@@ -1,5 +1,6 @@
 
 from ... schema import AgentRequest, AgentResponse
+from ... messaging import TranslatorRegistry
 
 from . requestor import ServiceRequestor
 
@@ -20,24 +21,12 @@ class AgentRequestor(ServiceRequestor):
             timeout=timeout,
         )
 
+        self.request_translator = TranslatorRegistry.get_request_translator("agent")
+        self.response_translator = TranslatorRegistry.get_response_translator("agent")
+
     def to_request(self, body):
-        return AgentRequest(
-            question=body["question"]
-        )
+        return self.request_translator.to_pulsar(body)
 
     def from_response(self, message):
-        resp = {
-        }
-
-        if message.answer:
-            resp["answer"] = message.answer
-
-        if message.thought:
-            resp["thought"] = message.thought
-
-        if message.observation:
-            resp["observation"] = message.observation
-
-        # The 2nd boolean expression indicates whether we're done responding
-        return resp, (message.answer is not None)
+        return self.response_translator.from_response_with_completion(message)
 

trustgraph-flow-1.0.17/trustgraph/gateway/dispatch/config.py

@@ -0,0 +1,31 @@
+
+from ... schema import ConfigRequest, ConfigResponse, ConfigKey, ConfigValue
+from ... schema import config_request_queue
+from ... schema import config_response_queue
+from ... messaging import TranslatorRegistry
+
+from . requestor import ServiceRequestor
+
+class ConfigRequestor(ServiceRequestor):
+    def __init__(self, pulsar_client, consumer, subscriber, timeout=120):
+
+        super(ConfigRequestor, self).__init__(
+            pulsar_client=pulsar_client,
+            consumer_name = consumer,
+            subscription = subscriber,
+            request_queue=config_request_queue,
+            response_queue=config_response_queue,
+            request_schema=ConfigRequest,
+            response_schema=ConfigResponse,
+            timeout=timeout,
+        )
+
+        self.request_translator = TranslatorRegistry.get_request_translator("config")
+        self.response_translator = TranslatorRegistry.get_response_translator("config")
+
+    def to_request(self, body):
+        return self.request_translator.to_pulsar(body)
+
+    def from_response(self, message):
+        return self.response_translator.from_response_with_completion(message)
+

{trustgraph-flow-1.0.15 → trustgraph-flow-1.0.17}/trustgraph/gateway/dispatch/document_embeddings_import.py

@@ -6,8 +6,7 @@ from aiohttp import WSMsgType
 from ... schema import Metadata
 from ... schema import DocumentEmbeddings, ChunkEmbeddings
 from ... base import Publisher
-
-from . serialize import to_subgraph
+from ... messaging.translators.document_loading import DocumentEmbeddingsTranslator
 
 class DocumentEmbeddingsImport:
 
@@ -17,6 +16,7 @@ class DocumentEmbeddingsImport:
 
         self.ws = ws
         self.running = running
+        self.translator = DocumentEmbeddingsTranslator()
         
         self.publisher = Publisher(
             pulsar_client, topic = queue, schema = DocumentEmbeddings
@@ -36,23 +36,7 @@ class DocumentEmbeddingsImport:
     async def receive(self, msg):
 
         data = msg.json()
-
-        elt = DocumentEmbeddings(
-            metadata=Metadata(
-                id=data["metadata"]["id"],
-                metadata=to_subgraph(data["metadata"]["metadata"]),
-                user=data["metadata"]["user"],
-                collection=data["metadata"]["collection"],
-            ),
-            chunks=[
-                ChunkEmbeddings(
-                    chunk=de["chunk"].encode("utf-8"),
-                    vectors=de["vectors"],
-                )
-                for de in data["chunks"]
-            ],
-        )
-
+        elt = self.translator.to_pulsar(data)
         await self.publisher.send(None, elt)
 
     async def run(self):

trustgraph-flow-1.0.17/trustgraph/gateway/dispatch/document_load.py

@@ -0,0 +1,23 @@
+
+import base64
+
+from ... schema import Document, Metadata
+from ... messaging import TranslatorRegistry
+
+from . sender import ServiceSender
+
+class DocumentLoad(ServiceSender):
+    def __init__(self, pulsar_client, queue):
+
+        super(DocumentLoad, self).__init__(
+            pulsar_client = pulsar_client,
+            queue = queue,
+            schema = Document,
+        )
+
+        self.translator = TranslatorRegistry.get_request_translator("document")
+
+    def to_request(self, body):
+        print("Document received")
+        return self.translator.to_pulsar(body)
+

{trustgraph-flow-1.0.15 → trustgraph-flow-1.0.17}/trustgraph/gateway/dispatch/document_rag.py

@@ -1,5 +1,6 @@
 
 from ... schema import DocumentRagQuery, DocumentRagResponse
+from ... messaging import TranslatorRegistry
 
 from . requestor import ServiceRequestor
 
@@ -20,14 +21,12 @@ class DocumentRagRequestor(ServiceRequestor):
             timeout=timeout,
         )
 
+        self.request_translator = TranslatorRegistry.get_request_translator("document-rag")
+        self.response_translator = TranslatorRegistry.get_response_translator("document-rag")
+
     def to_request(self, body):
-        return DocumentRagQuery(
-            query=body["query"],
-            user=body.get("user", "trustgraph"),
-            collection=body.get("collection", "default"),
-            doc_limit=int(body.get("doc-limit", 20)),
-        )
+        return self.request_translator.to_pulsar(body)
 
     def from_response(self, message):
-        return { "response": message.response }, True
+        return self.response_translator.from_response_with_completion(message)
 

{trustgraph-flow-1.0.15 → trustgraph-flow-1.0.17}/trustgraph/gateway/dispatch/embeddings.py

@@ -1,5 +1,6 @@
 
 from ... schema import EmbeddingsRequest, EmbeddingsResponse
+from ... messaging import TranslatorRegistry
 
 from . requestor import ServiceRequestor
 
@@ -20,11 +21,12 @@ class EmbeddingsRequestor(ServiceRequestor):
             timeout=timeout,
         )
 
+        self.request_translator = TranslatorRegistry.get_request_translator("embeddings")
+        self.response_translator = TranslatorRegistry.get_response_translator("embeddings")
+
     def to_request(self, body):
-        return EmbeddingsRequest(
-            text=body["text"]
-        )
+        return self.request_translator.to_pulsar(body)
 
     def from_response(self, message):
-        return { "vectors": message.vectors }, True
+        return self.response_translator.from_response_with_completion(message)
 

trustgraph-flow-1.0.17/trustgraph/gateway/dispatch/flow.py

@@ -0,0 +1,31 @@
+
+from ... schema import FlowRequest, FlowResponse
+from ... schema import flow_request_queue
+from ... schema import flow_response_queue
+from ... messaging import TranslatorRegistry
+
+from . requestor import ServiceRequestor
+
+class FlowRequestor(ServiceRequestor):
+    def __init__(self, pulsar_client, consumer, subscriber, timeout=120):
+
+        super(FlowRequestor, self).__init__(
+            pulsar_client=pulsar_client,
+            consumer_name = consumer,
+            subscription = subscriber,
+            request_queue=flow_request_queue,
+            response_queue=flow_response_queue,
+            request_schema=FlowRequest,
+            response_schema=FlowResponse,
+            timeout=timeout,
+        )
+
+        self.request_translator = TranslatorRegistry.get_request_translator("flow")
+        self.response_translator = TranslatorRegistry.get_response_translator("flow")
+
+    def to_request(self, body):
+        return self.request_translator.to_pulsar(body)
+
+    def from_response(self, message):
+        return self.response_translator.from_response_with_completion(message)
+

{trustgraph-flow-1.0.15 → trustgraph-flow-1.0.17}/trustgraph/gateway/dispatch/graph_embeddings_query.py

@@ -1,8 +1,8 @@
 
 from ... schema import GraphEmbeddingsRequest, GraphEmbeddingsResponse
+from ... messaging import TranslatorRegistry
 
 from . requestor import ServiceRequestor
-from . serialize import serialize_value
 
 class GraphEmbeddingsQueryRequestor(ServiceRequestor):
     def __init__(
@@ -21,22 +21,12 @@ class GraphEmbeddingsQueryRequestor(ServiceRequestor):
             timeout=timeout,
         )
 
-    def to_request(self, body):
-
-        limit = int(body.get("limit", 20))
+        self.request_translator = TranslatorRegistry.get_request_translator("graph-embeddings-query")
+        self.response_translator = TranslatorRegistry.get_response_translator("graph-embeddings-query")
 
-        return GraphEmbeddingsRequest(
-            vectors = body["vectors"],
-            limit = limit,
-            user = body.get("user", "trustgraph"),
-            collection = body.get("collection", "default"),
-        )
+    def to_request(self, body):
+        return self.request_translator.to_pulsar(body)
 
     def from_response(self, message):
-
-        return {
-            "entities": [
-                serialize_value(ent) for ent in message.entities
-            ]
-        }, True
+        return self.response_translator.from_response_with_completion(message)
 

{trustgraph-flow-1.0.15 → trustgraph-flow-1.0.17}/trustgraph/gateway/dispatch/graph_rag.py

@@ -1,5 +1,6 @@
 
 from ... schema import GraphRagQuery, GraphRagResponse
+from ... messaging import TranslatorRegistry
 
 from . requestor import ServiceRequestor
 
@@ -20,17 +21,12 @@ class GraphRagRequestor(ServiceRequestor):
             timeout=timeout,
         )
 
+        self.request_translator = TranslatorRegistry.get_request_translator("graph-rag")
+        self.response_translator = TranslatorRegistry.get_response_translator("graph-rag")
+
     def to_request(self, body):
-        return GraphRagQuery(
-            query=body["query"],
-            user=body.get("user", "trustgraph"),
-            collection=body.get("collection", "default"),
-            entity_limit=int(body.get("entity-limit", 50)),
-            triple_limit=int(body.get("triple-limit", 30)),
-            max_subgraph_size=int(body.get("max-subgraph-size", 1000)),
-            max_path_length=int(body.get("max-path-length", 2)),
-        )
+        return self.request_translator.to_pulsar(body)
 
     def from_response(self, message):
-        return { "response": message.response }, True
+        return self.response_translator.from_response_with_completion(message)
 

trustgraph-flow-1.0.17/trustgraph/gateway/dispatch/knowledge.py

@@ -0,0 +1,34 @@
+
+import base64
+
+from ... schema import KnowledgeRequest, KnowledgeResponse, Triples
+from ... schema import GraphEmbeddings, Metadata, EntityEmbeddings
+from ... schema import knowledge_request_queue
+from ... schema import knowledge_response_queue
+from ... messaging import TranslatorRegistry
+
+from . requestor import ServiceRequestor
+
+class KnowledgeRequestor(ServiceRequestor):
+    def __init__(self, pulsar_client, consumer, subscriber, timeout=120):
+
+        super(KnowledgeRequestor, self).__init__(
+            pulsar_client=pulsar_client,
+            consumer_name = consumer,
+            subscription = subscriber,
+            request_queue=knowledge_request_queue,
+            response_queue=knowledge_response_queue,
+            request_schema=KnowledgeRequest,
+            response_schema=KnowledgeResponse,
+            timeout=timeout,
+        )
+
+        self.request_translator = TranslatorRegistry.get_request_translator("knowledge")
+        self.response_translator = TranslatorRegistry.get_response_translator("knowledge")
+
+    def to_request(self, body):
+        return self.request_translator.to_pulsar(body)
+
+    def from_response(self, message):
+        return self.response_translator.from_response_with_completion(message)
+

trustgraph-flow-1.0.17/trustgraph/gateway/dispatch/librarian.py

@@ -0,0 +1,41 @@
+
+import base64
+
+from ... schema import LibrarianRequest, LibrarianResponse
+from ... schema import librarian_request_queue
+from ... schema import librarian_response_queue
+from ... messaging import TranslatorRegistry
+
+from . requestor import ServiceRequestor
+
+class LibrarianRequestor(ServiceRequestor):
+    def __init__(self, pulsar_client, consumer, subscriber, timeout=120):
+
+        super(LibrarianRequestor, self).__init__(
+            pulsar_client=pulsar_client,
+            consumer_name = consumer,
+            subscription = subscriber,
+            request_queue=librarian_request_queue,
+            response_queue=librarian_response_queue,
+            request_schema=LibrarianRequest,
+            response_schema=LibrarianResponse,
+            timeout=timeout,
+        )
+
+        self.request_translator = TranslatorRegistry.get_request_translator("librarian")
+        self.response_translator = TranslatorRegistry.get_response_translator("librarian")
+
+    def to_request(self, body):
+        # Handle base64 content processing
+        if "content" in body:
+            # Content gets base64 decoded & encoded again to ensure valid base64
+            content = base64.b64decode(body["content"].encode("utf-8"))
+            content = base64.b64encode(content).decode("utf-8")
+            body = body.copy()
+            body["content"] = content
+        
+        return self.request_translator.to_pulsar(body)
+
+    def from_response(self, message):
+        return self.response_translator.from_response_with_completion(message)
+

{trustgraph-flow-1.0.15 → trustgraph-flow-1.0.17}/trustgraph/gateway/dispatch/prompt.py

@@ -2,6 +2,7 @@
 import json
 
 from ... schema import PromptRequest, PromptResponse
+from ... messaging import TranslatorRegistry
 
 from . requestor import ServiceRequestor
 
@@ -22,22 +23,12 @@ class PromptRequestor(ServiceRequestor):
             timeout=timeout,
         )
 
+        self.request_translator = TranslatorRegistry.get_request_translator("prompt")
+        self.response_translator = TranslatorRegistry.get_response_translator("prompt")
+
     def to_request(self, body):
-        return PromptRequest(
-            id=body["id"],
-            terms={
-                k: json.dumps(v)
-                for k, v in body["variables"].items()
-            }
-        )
+        return self.request_translator.to_pulsar(body)
 
     def from_response(self, message):
-        if message.object:
-            return {
-                "object": message.object
-            }, True
-        else:
-            return {
-                "text": message.text
-            }, True
+        return self.response_translator.from_response_with_completion(message)
 

{trustgraph-flow-1.0.15 → trustgraph-flow-1.0.17}/trustgraph/gateway/dispatch/serialize.py

@@ -3,6 +3,13 @@ import base64
 
 from ... schema import Value, Triple, DocumentMetadata, ProcessingMetadata
 
+# DEPRECATED: These functions have been moved to trustgraph.... messaging.translators
+# Use the new messaging translation system instead for consistency and reusability.
+# Examples:
+#   from trustgraph.... messaging.translators.primitives import ValueTranslator
+#   value_translator = ValueTranslator()
+#   pulsar_value = value_translator.to_pulsar({"v": "example", "e": True})
+
 def to_value(x):
     return Value(value=x["v"], is_uri=x["e"])
 

{trustgraph-flow-1.0.15 → trustgraph-flow-1.0.17}/trustgraph/gateway/dispatch/text_completion.py

@@ -1,5 +1,6 @@
 
 from ... schema import TextCompletionRequest, TextCompletionResponse
+from ... messaging import TranslatorRegistry
 
 from . requestor import ServiceRequestor
 
@@ -20,12 +21,12 @@ class TextCompletionRequestor(ServiceRequestor):
             timeout=timeout,
         )
 
+        self.request_translator = TranslatorRegistry.get_request_translator("text-completion")
+        self.response_translator = TranslatorRegistry.get_response_translator("text-completion")
+
     def to_request(self, body):
-        return TextCompletionRequest(
-            system=body["system"],
-            prompt=body["prompt"]
-        )
+        return self.request_translator.to_pulsar(body)
 
     def from_response(self, message):
-        return { "response": message.response }, True
+        return self.response_translator.from_response_with_completion(message)
 

trustgraph-flow-1.0.17/trustgraph/gateway/dispatch/text_load.py

@@ -0,0 +1,23 @@
+
+import base64
+
+from ... schema import TextDocument, Metadata
+from ... messaging import TranslatorRegistry
+
+from . sender import ServiceSender
+
+class TextLoad(ServiceSender):
+    def __init__(self, pulsar_client, queue):
+
+        super(TextLoad, self).__init__(
+            pulsar_client = pulsar_client,
+            queue = queue,
+            schema = TextDocument,
+        )
+
+        self.translator = TranslatorRegistry.get_request_translator("text-document")
+
+    def to_request(self, body):
+        print("Text document received")
+        return self.translator.to_pulsar(body)
+

{trustgraph-flow-1.0.15 → trustgraph-flow-1.0.17}/trustgraph/gateway/dispatch/triples_query.py

@@ -1,8 +1,8 @@
 
 from ... schema import TriplesQueryRequest, TriplesQueryResponse, Triples
+from ... messaging import TranslatorRegistry
 
 from . requestor import ServiceRequestor
-from . serialize import to_value, serialize_subgraph
 
 class TriplesQueryRequestor(ServiceRequestor):
     def __init__(
@@ -21,34 +21,12 @@ class TriplesQueryRequestor(ServiceRequestor):
             timeout=timeout,
         )
 
-    def to_request(self, body):
-
-        if "s" in body:
-            s = to_value(body["s"])
-        else:
-            s = None
-
-        if "p" in body:
-            p = to_value(body["p"])
-        else:
-            p = None
-
-        if "o" in body:
-            o = to_value(body["o"])
-        else:
-            o = None
+        self.request_translator = TranslatorRegistry.get_request_translator("triples-query")
+        self.response_translator = TranslatorRegistry.get_response_translator("triples-query")
 
-        limit = int(body.get("limit", 10000))
-
-        return TriplesQueryRequest(
-            s = s, p = p, o = o,
-            limit = limit,
-            user = body.get("user", "trustgraph"),
-            collection = body.get("collection", "default"),
-        )
+    def to_request(self, body):
+        return self.request_translator.to_pulsar(body)
 
     def from_response(self, message):
-        return {
-            "response": serialize_subgraph(message.triples)
-        }, True
+        return self.response_translator.from_response_with_completion(message)
 

{trustgraph-flow-1.0.15 → trustgraph-flow-1.0.17}/trustgraph_flow.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 1.0.15
+Version: 1.0.17
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v1.0.15.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v1.0.17.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

trustgraph-flow-1.0.15/trustgraph/flow_version.py

@@ -1 +0,0 @@
-__version__ = "1.0.15"

trustgraph-flow-1.0.15/trustgraph/gateway/dispatch/config.py

@@ -1,78 +0,0 @@
-
-from ... schema import ConfigRequest, ConfigResponse, ConfigKey, ConfigValue
-from ... schema import config_request_queue
-from ... schema import config_response_queue
-
-from . requestor import ServiceRequestor
-
-class ConfigRequestor(ServiceRequestor):
-    def __init__(self, pulsar_client, consumer, subscriber, timeout=120):
-
-        super(ConfigRequestor, self).__init__(
-            pulsar_client=pulsar_client,
-            consumer_name = consumer,
-            subscription = subscriber,
-            request_queue=config_request_queue,
-            response_queue=config_response_queue,
-            request_schema=ConfigRequest,
-            response_schema=ConfigResponse,
-            timeout=timeout,
-        )
-
-    def to_request(self, body):
-
-        if "keys" in body:
-            keys = [
-                ConfigKey(
-                    type = k["type"],
-                    key = k["key"],
-                )
-                for k in body["keys"]
-            ]
-        else:
-            keys = None
-
-        if "values" in body:
-            values = [
-                ConfigValue(
-                    type = v["type"],
-                    key = v["key"],
-                    value = v["value"],
-                )
-                for v in body["values"]
-            ]
-        else:
-            values = None
-
-        return ConfigRequest(
-            operation = body.get("operation", None),
-            keys = keys,
-            type = body.get("type", None),
-            values = values
-        )
-
-    def from_response(self, message):
-
-        response = { }
-
-        if message.version is not None:
-            response["version"] = message.version
-
-        if message.values is not None:
-            response["values"] = [
-                {
-                    "type": v.type,
-                    "key": v.key,
-                    "value": v.value,
-                }
-                for v in message.values
-            ]
-
-        if message.directory is not None:
-            response["directory"] = message.directory
-
-        if message.config is not None:
-            response["config"] = message.config
-
-        return response, True
-

trustgraph-flow-1.0.15/trustgraph/gateway/dispatch/document_load.py

@@ -1,40 +0,0 @@
-
-import base64
-
-from ... schema import Document, Metadata
-
-from . sender import ServiceSender
-from . serialize import to_subgraph
-
-class DocumentLoad(ServiceSender):
-    def __init__(self, pulsar_client, queue):
-
-        super(DocumentLoad, self).__init__(
-            pulsar_client = pulsar_client,
-            queue = queue,
-            schema = Document,
-        )
-
-    def to_request(self, body):
-
-        if "metadata" in body:
-            metadata = to_subgraph(body["metadata"])
-        else:
-            metadata = []
-
-        # Doing a base64 decoe/encode here to make sure the
-        # content is valid base64
-        doc = base64.b64decode(body["data"])
-
-        print("Document received")
-
-        return Document(
-            metadata=Metadata(
-                id=body.get("id"),
-                metadata=metadata,
-                user=body.get("user", "trustgraph"),
-                collection=body.get("collection", "default"),
-            ),
-            data=base64.b64encode(doc).decode("utf-8")
-        )
-