trustgraph 0.5.3__tar.gz → 0.11.20__tar.gz

trustgraph-0.11.20/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.1
+Name: trustgraph
+Version: 0.11.20
+Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
+Home-page: https://github.com/trustgraph-ai/trustgraph
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.11.20.tar.gz
+Author: trustgraph.ai
+Author-email: security@trustgraph.ai
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: trustgraph-base<0.12
+Requires-Dist: trustgraph-bedrock<0.12
+Requires-Dist: trustgraph-cli<0.12
+Requires-Dist: trustgraph-embeddings-hf<0.12
+Requires-Dist: trustgraph-flow<0.12
+Requires-Dist: trustgraph-parquet<0.12
+Requires-Dist: trustgraph-vertexai<0.12
+
+See https://trustgraph.ai/

trustgraph-0.11.20/README.md

@@ -0,0 +1 @@
+See https://trustgraph.ai/

trustgraph-0.11.20/setup.py

@@ -0,0 +1,47 @@
+import setuptools
+import os
+import importlib
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+# Load a version number module
+spec = importlib.util.spec_from_file_location(
+    'version', 'trustgraph/trustgraph_version.py'
+)
+version_module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(version_module)
+
+version = version_module.__version__
+
+setuptools.setup(
+    name="trustgraph",
+    version=version,
+    author="trustgraph.ai",
+    author_email="security@trustgraph.ai",
+    description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/trustgraph-ai/trustgraph",
+    packages=setuptools.find_namespace_packages(
+        where='./',
+    ),
+    classifiers=[ 
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+        "Operating System :: OS Independent",
+    ],
+    python_requires='>=3.8',
+    download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
+    install_requires=[
+        "trustgraph-base<0.12",
+        "trustgraph-bedrock<0.12",
+        "trustgraph-cli<0.12",
+        "trustgraph-embeddings-hf<0.12",
+        "trustgraph-flow<0.12",
+        "trustgraph-parquet<0.12",
+        "trustgraph-vertexai<0.12",
+    ],
+    scripts=[
+    ]
+)

{trustgraph-0.5.3 → trustgraph-0.11.20}/trustgraph/base/base_processor.py

@@ -60,10 +60,10 @@ class BaseProcessor:
         )
 
         parser.add_argument(
-            '-M', '--metrics-enabled',
-            type=bool,
+            '--metrics',
+            action=argparse.BooleanOptionalAction,
             default=True,
-            help=f'Pulsar host (default: true)',
+            help=f'Metrics enabled (default: true)',
         )
 
         parser.add_argument(
@@ -89,7 +89,9 @@ class BaseProcessor:
         args = parser.parse_args()
         args = vars(args)
 
-        if args["metrics_enabled"]:
+        print(args)
+
+        if args["metrics"]:
             start_http_server(args["metrics_port"])
 
         while True:
@@ -114,4 +116,4 @@ class BaseProcessor:
                 print("Exception:", e, flush=True)
                 print("Will retry...", flush=True)
 
-                time.sleep(10)
+                time.sleep(4)

{trustgraph-0.5.3 → trustgraph-0.11.20}/trustgraph/base/consumer.py

@@ -1,13 +1,24 @@
 
 from pulsar.schema import JsonSchema
-from prometheus_client import start_http_server, Histogram, Info, Counter
+from prometheus_client import Histogram, Info, Counter, Enum
+import time
 
 from . base_processor import BaseProcessor
+from .. exceptions import TooManyRequests
 
 class Consumer(BaseProcessor):
 
     def __init__(self, **params):
 
+        if not hasattr(__class__, "state_metric"):
+            __class__.state_metric = Enum(
+                'processor_state', 'Processor state',
+                states=['starting', 'running', 'stopped']
+            )
+            __class__.state_metric.state('starting')
+
+        __class__.state_metric.state('starting')
+
         super(Consumer, self).__init__(**params)
 
         input_queue = params.get("input_queue")
@@ -45,6 +56,8 @@ class Consumer(BaseProcessor):
 
     def run(self):
 
+        __class__.state_metric.state('running')
+
         while True:
 
             msg = self.consumer.receive()
@@ -59,6 +72,13 @@ class Consumer(BaseProcessor):
 
                 __class__.processing_metric.labels(status="success").inc()
 
+            except TooManyRequests:
+                self.consumer.negative_acknowledge(msg)
+                print("TooManyRequests: will retry")
+                __class__.processing_metric.labels(status="rate-limit").inc()
+                time.sleep(5)
+                continue
+                
             except Exception as e:
 
                 print("Exception:", e, flush=True)

{trustgraph-0.5.3 → trustgraph-0.11.20}/trustgraph/base/consumer_producer.py

@@ -1,8 +1,10 @@
 
 from pulsar.schema import JsonSchema
-from prometheus_client import Histogram, Info, Counter
+from prometheus_client import Histogram, Info, Counter, Enum
+import time
 
 from . base_processor import BaseProcessor
+from .. exceptions import TooManyRequests
 
 # FIXME: Derive from consumer?  And producer?
 
@@ -10,6 +12,15 @@ class ConsumerProducer(BaseProcessor):
 
     def __init__(self, **params):
 
+        if not hasattr(__class__, "state_metric"):
+            __class__.state_metric = Enum(
+                'processor_state', 'Processor state',
+                states=['starting', 'running', 'stopped']
+            )
+            __class__.state_metric.state('starting')
+
+        __class__.state_metric.state('starting')
+
         input_queue = params.get("input_queue")
         output_queue = params.get("output_queue")
         subscriber = params.get("subscriber")
@@ -52,18 +63,20 @@ class ConsumerProducer(BaseProcessor):
         if output_schema == None:
             raise RuntimeError("output_schema must be specified")
 
-        self.consumer = self.client.subscribe(
-            input_queue, subscriber,
-            schema=JsonSchema(input_schema),
-        )
-
         self.producer = self.client.create_producer(
             topic=output_queue,
             schema=JsonSchema(output_schema),
         )
 
+        self.consumer = self.client.subscribe(
+            input_queue, subscriber,
+            schema=JsonSchema(input_schema),
+        )
+
     def run(self):
 
+        __class__.state_metric.state('running')
+
         while True:
 
             msg = self.consumer.receive()
@@ -78,6 +91,13 @@ class ConsumerProducer(BaseProcessor):
 
                 __class__.processing_metric.labels(status="success").inc()
 
+            except TooManyRequests:
+                self.consumer.negative_acknowledge(msg)
+                print("TooManyRequests: will retry")
+                __class__.processing_metric.labels(status="rate-limit").inc()
+                time.sleep(5)
+                continue
+
             except Exception as e:
 
                 print("Exception:", e, flush=True)
@@ -117,52 +137,3 @@ class ConsumerProducer(BaseProcessor):
             help=f'Output queue (default: {default_output_queue})'
         )
 
-class Producer(BaseProcessor):
-
-    def __init__(self, **params):
-
-        output_queue = params.get("output_queue")
-        output_schema = params.get("output_schema")
-
-        if not hasattr(__class__, "output_metric"):
-            __class__.output_metric = Counter(
-                'output_count', 'Output items created'
-            )
-
-        if not hasattr(__class__, "pubsub_metric"):
-            __class__.pubsub_metric = Info(
-                'pubsub', 'Pub/sub configuration'
-            )
-
-        __class__.pubsub_metric.info({
-            "output_queue": output_queue,
-            "output_schema": output_schema.__name__,
-        })
-
-        super(Producer, self).__init__(**params)
-
-        if output_schema == None:
-            raise RuntimeError("output_schema must be specified")
-
-        self.producer = self.client.create_producer(
-            topic=output_queue,
-            schema=JsonSchema(output_schema),
-        )
-
-    def send(self, msg, properties={}):
-        self.producer.send(msg, properties)
-        __class__.output_metric.inc()
-
-    @staticmethod
-    def add_args(
-            parser, default_input_queue, default_subscriber,
-            default_output_queue,
-    ):
-
-        BaseProcessor.add_args(parser)
-
-        parser.add_argument(
-            '-o', '--output-queue',
-            default=default_output_queue,
-            help=f'Output queue (default: {default_output_queue})'
-        )

trustgraph-0.11.20/trustgraph/base_version.py

@@ -0,0 +1 @@
+__version__ = "0.12.0"

trustgraph-0.11.20/trustgraph/clients/base.py

@@ -0,0 +1,125 @@
+
+import pulsar
+import _pulsar
+import hashlib
+import uuid
+import time
+from pulsar.schema import JsonSchema
+
+from .. exceptions import *
+
+# Default timeout for a request/response.  In seconds.
+DEFAULT_TIMEOUT=300
+
+# Ugly
+ERROR=_pulsar.LoggerLevel.Error
+WARN=_pulsar.LoggerLevel.Warn
+INFO=_pulsar.LoggerLevel.Info
+DEBUG=_pulsar.LoggerLevel.Debug
+
+class BaseClient:
+
+    def __init__(
+            self, log_level=ERROR,
+            subscriber=None,
+            input_queue=None,
+            output_queue=None,
+            input_schema=None,
+            output_schema=None,
+            pulsar_host="pulsar://pulsar:6650",
+    ):
+
+        if input_queue == None: raise RuntimeError("Need input_queue")
+        if output_queue == None: raise RuntimeError("Need output_queue")
+        if input_schema == None: raise RuntimeError("Need input_schema")
+        if output_schema == None: raise RuntimeError("Need output_schema")
+
+        if subscriber == None:
+            subscriber = str(uuid.uuid4())
+
+        self.client = pulsar.Client(
+            pulsar_host,
+            logger=pulsar.ConsoleLogger(log_level),
+        )
+
+        self.producer = self.client.create_producer(
+            topic=input_queue,
+            schema=JsonSchema(input_schema),
+            chunking_enabled=True,
+        )
+
+        self.consumer = self.client.subscribe(
+            output_queue, subscriber,
+            schema=JsonSchema(output_schema),
+        )
+
+        self.input_schema = input_schema
+        self.output_schema = output_schema
+
+    def call(self, **args):
+
+        timeout = args.get("timeout", DEFAULT_TIMEOUT)
+
+        if "timeout" in args:
+            del args["timeout"]
+
+        id = str(uuid.uuid4())
+
+        r = self.input_schema(**args)
+
+        end_time = time.time() + timeout
+
+        self.producer.send(r, properties={ "id": id })
+
+        while time.time() < end_time:
+
+            try:
+                msg = self.consumer.receive(timeout_millis=2500)
+            except pulsar.exceptions.Timeout:
+                continue
+
+            mid = msg.properties()["id"]
+
+            if mid == id:
+
+                value = msg.value()
+
+                if value.error:
+
+                    self.consumer.acknowledge(msg)
+
+                    if value.error.type == "llm-error":
+                        raise LlmError(value.error.message)
+
+                    elif value.error.type == "too-many-requests":
+                        raise TooManyRequests(value.error.message)
+
+                    elif value.error.type == "ParseError":
+                        raise ParseError(value.error.message)
+
+                    else:
+
+                        raise RuntimeError(
+                            f"{value.error.type}: {value.error.message}"
+                        )
+
+                resp = msg.value()
+                self.consumer.acknowledge(msg)
+                return resp
+
+            # Ignore messages with wrong ID
+            self.consumer.acknowledge(msg)
+
+        raise TimeoutError("Timed out waiting for response")
+
+    def __del__(self):
+
+        if hasattr(self, "consumer"):
+            self.consumer.close()
+            
+        if hasattr(self, "producer"):
+            self.producer.flush()
+            self.producer.close()
+            
+        self.client.close()
+

trustgraph-0.11.20/trustgraph/clients/document_embeddings_client.py

@@ -0,0 +1,45 @@
+
+import _pulsar
+
+from .. schema import DocumentEmbeddingsRequest, DocumentEmbeddingsResponse
+from .. schema import document_embeddings_request_queue
+from .. schema import document_embeddings_response_queue
+from . base import BaseClient
+
+# Ugly
+ERROR=_pulsar.LoggerLevel.Error
+WARN=_pulsar.LoggerLevel.Warn
+INFO=_pulsar.LoggerLevel.Info
+DEBUG=_pulsar.LoggerLevel.Debug
+
+class DocumentEmbeddingsClient(BaseClient):
+
+    def __init__(
+            self, log_level=ERROR,
+            subscriber=None,
+            input_queue=None,
+            output_queue=None,
+            pulsar_host="pulsar://pulsar:6650",
+    ):
+
+        if input_queue == None:
+            input_queue = document_embeddings_request_queue
+
+        if output_queue == None:
+            output_queue = document_embeddings_response_queue
+            
+        super(DocumentEmbeddingsClient, self).__init__(
+            log_level=log_level,
+            subscriber=subscriber,
+            input_queue=input_queue,
+            output_queue=output_queue,
+            pulsar_host=pulsar_host,
+            input_schema=DocumentEmbeddingsRequest,
+            output_schema=DocumentEmbeddingsResponse,
+        )
+
+    def request(self, vectors, limit=10, timeout=300):
+        return self.call(
+            vectors=vectors, limit=limit, timeout=timeout
+        ).documents
+

trustgraph-0.11.20/trustgraph/clients/document_rag_client.py

@@ -0,0 +1,46 @@
+
+import _pulsar
+
+from .. schema import DocumentRagQuery, DocumentRagResponse
+from .. schema import document_rag_request_queue, document_rag_response_queue
+from . base import BaseClient
+
+# Ugly
+ERROR=_pulsar.LoggerLevel.Error
+WARN=_pulsar.LoggerLevel.Warn
+INFO=_pulsar.LoggerLevel.Info
+DEBUG=_pulsar.LoggerLevel.Debug
+
+class DocumentRagClient(BaseClient):
+
+    def __init__(
+            self,
+            log_level=ERROR,
+            subscriber=None,
+            input_queue=None,
+            output_queue=None,
+            pulsar_host="pulsar://pulsar:6650",
+    ):
+
+        if input_queue == None:
+            input_queue = document_rag_request_queue
+
+        if output_queue == None:
+            output_queue = document_rag_response_queue
+  
+        super(DocumentRagClient, self).__init__(
+            log_level=log_level,
+            subscriber=subscriber,
+            input_queue=input_queue,
+            output_queue=output_queue,
+            pulsar_host=pulsar_host,
+            input_schema=DocumentRagQuery,
+            output_schema=DocumentRagResponse,
+        )
+
+    def request(self, query, timeout=500):
+
+        return self.call(
+            query=query, timeout=timeout
+        ).response
+

trustgraph-0.11.20/trustgraph/clients/embeddings_client.py

@@ -0,0 +1,44 @@
+
+from pulsar.schema import JsonSchema
+from .. schema import EmbeddingsRequest, EmbeddingsResponse
+from .. schema import embeddings_request_queue, embeddings_response_queue
+from . base import BaseClient
+
+import _pulsar
+
+# Ugly
+ERROR=_pulsar.LoggerLevel.Error
+WARN=_pulsar.LoggerLevel.Warn
+INFO=_pulsar.LoggerLevel.Info
+DEBUG=_pulsar.LoggerLevel.Debug
+
+class EmbeddingsClient(BaseClient):
+
+    def __init__(
+            self, log_level=ERROR,
+            input_queue=None,
+            output_queue=None,
+            subscriber=None,
+            pulsar_host="pulsar://pulsar:6650",
+    ):
+
+        if input_queue == None:
+            input_queue=embeddings_request_queue
+
+        if output_queue == None:
+            output_queue=embeddings_response_queue
+
+        super(EmbeddingsClient, self).__init__(
+            log_level=log_level,
+            subscriber=subscriber,
+            input_queue=input_queue,
+            output_queue=output_queue,
+            pulsar_host=pulsar_host,
+            input_schema=EmbeddingsRequest,
+            output_schema=EmbeddingsResponse,
+        )
+
+    def request(self, text, timeout=300):
+        return self.call(text=text, timeout=timeout).vectors
+
+

trustgraph-0.11.20/trustgraph/clients/graph_embeddings_client.py

@@ -0,0 +1,45 @@
+
+import _pulsar
+
+from .. schema import GraphEmbeddingsRequest, GraphEmbeddingsResponse
+from .. schema import graph_embeddings_request_queue
+from .. schema import graph_embeddings_response_queue
+from . base import BaseClient
+
+# Ugly
+ERROR=_pulsar.LoggerLevel.Error
+WARN=_pulsar.LoggerLevel.Warn
+INFO=_pulsar.LoggerLevel.Info
+DEBUG=_pulsar.LoggerLevel.Debug
+
+class GraphEmbeddingsClient(BaseClient):
+
+    def __init__(
+            self, log_level=ERROR,
+            subscriber=None,
+            input_queue=None,
+            output_queue=None,
+            pulsar_host="pulsar://pulsar:6650",
+    ):
+
+        if input_queue == None:
+            input_queue = graph_embeddings_request_queue
+
+        if output_queue == None:
+            output_queue = graph_embeddings_response_queue
+            
+        super(GraphEmbeddingsClient, self).__init__(
+            log_level=log_level,
+            subscriber=subscriber,
+            input_queue=input_queue,
+            output_queue=output_queue,
+            pulsar_host=pulsar_host,
+            input_schema=GraphEmbeddingsRequest,
+            output_schema=GraphEmbeddingsResponse,
+        )
+
+    def request(self, vectors, limit=10, timeout=300):
+        return self.call(
+            vectors=vectors, limit=limit, timeout=timeout
+        ).entities
+

trustgraph-0.11.20/trustgraph/clients/graph_rag_client.py

@@ -0,0 +1,46 @@
+
+import _pulsar
+
+from .. schema import GraphRagQuery, GraphRagResponse
+from .. schema import graph_rag_request_queue, graph_rag_response_queue
+from . base import BaseClient
+
+# Ugly
+ERROR=_pulsar.LoggerLevel.Error
+WARN=_pulsar.LoggerLevel.Warn
+INFO=_pulsar.LoggerLevel.Info
+DEBUG=_pulsar.LoggerLevel.Debug
+
+class GraphRagClient(BaseClient):
+
+    def __init__(
+            self,
+            log_level=ERROR,
+            subscriber=None,
+            input_queue=None,
+            output_queue=None,
+            pulsar_host="pulsar://pulsar:6650",
+    ):
+
+        if input_queue == None:
+            input_queue = graph_rag_request_queue
+
+        if output_queue == None:
+            output_queue = graph_rag_response_queue
+  
+        super(GraphRagClient, self).__init__(
+            log_level=log_level,
+            subscriber=subscriber,
+            input_queue=input_queue,
+            output_queue=output_queue,
+            pulsar_host=pulsar_host,
+            input_schema=GraphRagQuery,
+            output_schema=GraphRagResponse,
+        )
+
+    def request(self, query, timeout=500):
+
+        return self.call(
+            query=query, timeout=timeout
+        ).response
+

trustgraph-0.11.20/trustgraph/clients/llm_client.py

@@ -0,0 +1,40 @@
+
+import _pulsar
+
+from .. schema import TextCompletionRequest, TextCompletionResponse
+from .. schema import text_completion_request_queue
+from .. schema import text_completion_response_queue
+from . base import BaseClient
+
+# Ugly
+ERROR=_pulsar.LoggerLevel.Error
+WARN=_pulsar.LoggerLevel.Warn
+INFO=_pulsar.LoggerLevel.Info
+DEBUG=_pulsar.LoggerLevel.Debug
+
+class LlmClient(BaseClient):
+
+    def __init__(
+            self, log_level=ERROR,
+            subscriber=None,
+            input_queue=None,
+            output_queue=None,
+            pulsar_host="pulsar://pulsar:6650",
+    ):
+
+        if input_queue is None: input_queue = text_completion_request_queue
+        if output_queue is None: output_queue = text_completion_response_queue
+
+        super(LlmClient, self).__init__(
+            log_level=log_level,
+            subscriber=subscriber,
+            input_queue=input_queue,
+            output_queue=output_queue,
+            pulsar_host=pulsar_host,
+            input_schema=TextCompletionRequest,
+            output_schema=TextCompletionResponse,
+        )
+
+    def request(self, prompt, timeout=300):
+        return self.call(prompt=prompt, timeout=timeout).response
+