trustgraph-base 0.20.1__tar.gz → 0.20.2__tar.gz

{trustgraph-base-0.20.1 → trustgraph-base-0.20.2}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-base
-Version: 0.20.1
+Version: 0.20.2
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.20.1.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.20.2.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

trustgraph-base-0.20.2/trustgraph/base/consumer.py

@@ -0,0 +1,172 @@
+
+from pulsar.schema import JsonSchema
+import pulsar
+from prometheus_client import Histogram, Info, Counter, Enum
+import time
+
+from . base_processor import BaseProcessor
+from .. exceptions import TooManyRequests
+
+default_rate_limit_retry = 10
+default_rate_limit_timeout = 7200
+
+class Consumer(BaseProcessor):
+
+    def __init__(self, **params):
+
+        if not hasattr(__class__, "state_metric"):
+            __class__.state_metric = Enum(
+                'processor_state', 'Processor state',
+                states=['starting', 'running', 'stopped']
+            )
+            __class__.state_metric.state('starting')
+
+        __class__.state_metric.state('starting')
+
+        super(Consumer, self).__init__(**params)
+
+        self.input_queue = params.get("input_queue")
+        self.subscriber = params.get("subscriber")
+        self.input_schema = params.get("input_schema")
+
+        self.rate_limit_retry = params.get(
+            "rate_limit_retry", default_rate_limit_retry
+        )
+        self.rate_limit_timeout = params.get(
+            "rate_limit_timeout", default_rate_limit_timeout
+        )
+
+        if self.input_schema == None:
+            raise RuntimeError("input_schema must be specified")
+
+        if not hasattr(__class__, "request_metric"):
+            __class__.request_metric = Histogram(
+                'request_latency', 'Request latency (seconds)'
+            )
+
+        if not hasattr(__class__, "pubsub_metric"):
+            __class__.pubsub_metric = Info(
+                'pubsub', 'Pub/sub configuration'
+            )
+
+        if not hasattr(__class__, "processing_metric"):
+            __class__.processing_metric = Counter(
+                'processing_count', 'Processing count', ["status"]
+            )
+
+        if not hasattr(__class__, "rate_limit_metric"):
+            __class__.rate_limit_metric = Counter(
+                'rate_limit_count', 'Rate limit event count',
+            )
+
+        __class__.pubsub_metric.info({
+            "input_queue": self.input_queue,
+            "subscriber": self.subscriber,
+            "input_schema": self.input_schema.__name__,
+            "rate_limit_retry": str(self.rate_limit_retry),
+            "rate_limit_timeout": str(self.rate_limit_timeout),
+        })
+
+        self.consumer = self.client.subscribe(
+            self.input_queue, self.subscriber,
+            consumer_type=pulsar.ConsumerType.Shared,
+            schema=JsonSchema(self.input_schema),
+        )
+
+        print("Initialised consumer.", flush=True)
+
+    def run(self):
+
+        __class__.state_metric.state('running')
+
+        while True:
+
+            msg = self.consumer.receive()
+
+            expiry = time.time() + self.rate_limit_timeout
+
+            # This loop is for retry on rate-limit / resource limits
+            while True:
+
+                if time.time() > expiry:
+
+                    print("Gave up waiting for rate-limit retry", flush=True)
+
+                    # Message failed to be processed, this causes it to
+                    # be retried
+                    self.consumer.negative_acknowledge(msg)
+
+                    __class__.processing_metric.labels(status="error").inc()
+
+                    # Break out of retry loop, processes next message
+                    break
+
+                try:
+
+                    with __class__.request_metric.time():
+                        self.handle(msg)
+
+                    # Acknowledge successful processing of the message
+                    self.consumer.acknowledge(msg)
+
+                    __class__.processing_metric.labels(status="success").inc()
+
+                    # Break out of retry loop
+                    break
+
+                except TooManyRequests:
+
+                    print("TooManyRequests: will retry...", flush=True)
+
+                    __class__.rate_limit_metric.inc()
+
+                    # Sleep
+                    time.sleep(self.rate_limit_retry)
+
+                    # Contine from retry loop, just causes a reprocessing
+                    continue
+                
+                except Exception as e:
+
+                    print("Exception:", e, flush=True)
+
+                    # Message failed to be processed, this causes it to
+                    # be retried
+                    self.consumer.negative_acknowledge(msg)
+
+                    __class__.processing_metric.labels(status="error").inc()
+
+                    # Break out of retry loop, processes next message
+                    break
+
+    @staticmethod
+    def add_args(parser, default_input_queue, default_subscriber):
+
+        BaseProcessor.add_args(parser)
+
+        parser.add_argument(
+            '-i', '--input-queue',
+            default=default_input_queue,
+            help=f'Input queue (default: {default_input_queue})'
+        )
+
+        parser.add_argument(
+            '-s', '--subscriber',
+            default=default_subscriber,
+            help=f'Queue subscriber name (default: {default_subscriber})'
+        )
+
+        parser.add_argument(
+            '--rate-limit-retry',
+            type=int,
+            default=default_rate_limit_retry,
+            help=f'Rate limit retry (default: {default_rate_limit_retry})'
+        )
+
+        parser.add_argument(
+            '--rate-limit-timeout',
+            type=int,
+            default=default_rate_limit_timeout,
+            help=f'Rate limit timeout (default: {default_rate_limit_timeout})'
+        )
+

trustgraph-base-0.20.2/trustgraph/base/consumer_producer.py

@@ -0,0 +1,62 @@
+
+from pulsar.schema import JsonSchema
+import pulsar
+from prometheus_client import Histogram, Info, Counter, Enum
+import time
+
+from . consumer import Consumer
+from .. exceptions import TooManyRequests
+
+class ConsumerProducer(Consumer):
+
+    def __init__(self, **params):
+
+        super(ConsumerProducer, self).__init__(**params)
+
+        self.output_queue = params.get("output_queue")
+        self.output_schema = params.get("output_schema")
+
+        if not hasattr(__class__, "output_metric"):
+            __class__.output_metric = Counter(
+                'output_count', 'Output items created'
+            )
+
+        __class__.pubsub_metric.info({
+            "input_queue": self.input_queue,
+            "output_queue": self.output_queue,
+            "subscriber": self.subscriber,
+            "input_schema": self.input_schema.__name__,
+            "output_schema": self.output_schema.__name__,
+            "rate_limit_retry": str(self.rate_limit_retry),
+            "rate_limit_timeout": str(self.rate_limit_timeout),
+        })
+
+        if self.output_schema == None:
+            raise RuntimeError("output_schema must be specified")
+
+        self.producer = self.client.create_producer(
+            topic=self.output_queue,
+            schema=JsonSchema(self.output_schema),
+            chunking_enabled=True,
+        )
+
+        print("Initialised consumer/producer.")
+
+    def send(self, msg, properties={}):
+        self.producer.send(msg, properties)
+        __class__.output_metric.inc()
+
+    @staticmethod
+    def add_args(
+            parser, default_input_queue, default_subscriber,
+            default_output_queue,
+    ):
+
+        Consumer.add_args(parser, default_input_queue, default_subscriber)
+
+        parser.add_argument(
+            '-o', '--output-queue',
+            default=default_output_queue,
+            help=f'Output queue (default: {default_output_queue})'
+        )
+

trustgraph-base-0.20.2/trustgraph/base_version.py

@@ -0,0 +1 @@
+__version__ = "0.20.2"

{trustgraph-base-0.20.1 → trustgraph-base-0.20.2}/trustgraph/exceptions.py

@@ -8,7 +8,3 @@ class LlmError(Exception):
 class ParseError(Exception):
     pass
 
-
-
-
-

{trustgraph-base-0.20.1 → trustgraph-base-0.20.2}/trustgraph_base.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-base
-Version: 0.20.1
+Version: 0.20.2
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.20.1.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.20.2.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

trustgraph-base-0.20.1/trustgraph/base/consumer.py

@@ -1,109 +0,0 @@
-
-from pulsar.schema import JsonSchema
-import pulsar
-from prometheus_client import Histogram, Info, Counter, Enum
-import time
-
-from . base_processor import BaseProcessor
-from .. exceptions import TooManyRequests
-
-class Consumer(BaseProcessor):
-
-    def __init__(self, **params):
-
-        if not hasattr(__class__, "state_metric"):
-            __class__.state_metric = Enum(
-                'processor_state', 'Processor state',
-                states=['starting', 'running', 'stopped']
-            )
-            __class__.state_metric.state('starting')
-
-        __class__.state_metric.state('starting')
-
-        super(Consumer, self).__init__(**params)
-
-        input_queue = params.get("input_queue")
-        subscriber = params.get("subscriber")
-        input_schema = params.get("input_schema")
-
-        if input_schema == None:
-            raise RuntimeError("input_schema must be specified")
-
-        if not hasattr(__class__, "request_metric"):
-            __class__.request_metric = Histogram(
-                'request_latency', 'Request latency (seconds)'
-            )
-
-        if not hasattr(__class__, "pubsub_metric"):
-            __class__.pubsub_metric = Info(
-                'pubsub', 'Pub/sub configuration'
-            )
-
-        if not hasattr(__class__, "processing_metric"):
-            __class__.processing_metric = Counter(
-                'processing_count', 'Processing count', ["status"]
-            )
-
-        __class__.pubsub_metric.info({
-            "input_queue": input_queue,
-            "subscriber": subscriber,
-            "input_schema": input_schema.__name__,
-        })
-
-        self.consumer = self.client.subscribe(
-            input_queue, subscriber,
-            consumer_type=pulsar.ConsumerType.Shared,
-            schema=JsonSchema(input_schema),
-        )
-
-    def run(self):
-
-        __class__.state_metric.state('running')
-
-        while True:
-
-            msg = self.consumer.receive()
-
-            try:
-
-                with __class__.request_metric.time():
-                    self.handle(msg)
-
-                # Acknowledge successful processing of the message
-                self.consumer.acknowledge(msg)
-
-                __class__.processing_metric.labels(status="success").inc()
-
-            except TooManyRequests:
-                self.consumer.negative_acknowledge(msg)
-                print("TooManyRequests: will retry")
-                __class__.processing_metric.labels(status="rate-limit").inc()
-                time.sleep(5)
-                continue
-                
-            except Exception as e:
-
-                print("Exception:", e, flush=True)
-
-                # Message failed to be processed
-                self.consumer.negative_acknowledge(msg)
-
-                __class__.processing_metric.labels(status="error").inc()
-
-    @staticmethod
-    def add_args(parser, default_input_queue, default_subscriber):
-
-        BaseProcessor.add_args(parser)
-
-        parser.add_argument(
-            '-i', '--input-queue',
-            default=default_input_queue,
-            help=f'Input queue (default: {default_input_queue})'
-        )
-
-        parser.add_argument(
-            '-s', '--subscriber',
-            default=default_subscriber,
-            help=f'Queue subscriber name (default: {default_subscriber})'
-        )
-

trustgraph-base-0.20.1/trustgraph/base/consumer_producer.py

@@ -1,142 +0,0 @@
-
-from pulsar.schema import JsonSchema
-import pulsar
-from prometheus_client import Histogram, Info, Counter, Enum
-import time
-
-from . base_processor import BaseProcessor
-from .. exceptions import TooManyRequests
-
-# FIXME: Derive from consumer?  And producer?
-
-class ConsumerProducer(BaseProcessor):
-
-    def __init__(self, **params):
-
-        if not hasattr(__class__, "state_metric"):
-            __class__.state_metric = Enum(
-                'processor_state', 'Processor state',
-                states=['starting', 'running', 'stopped']
-            )
-            __class__.state_metric.state('starting')
-
-        __class__.state_metric.state('starting')
-
-        input_queue = params.get("input_queue")
-        output_queue = params.get("output_queue")
-        subscriber = params.get("subscriber")
-        input_schema = params.get("input_schema")
-        output_schema = params.get("output_schema")
-
-        if not hasattr(__class__, "request_metric"):
-            __class__.request_metric = Histogram(
-                'request_latency', 'Request latency (seconds)'
-            )
-
-        if not hasattr(__class__, "output_metric"):
-            __class__.output_metric = Counter(
-                'output_count', 'Output items created'
-            )
-
-        if not hasattr(__class__, "pubsub_metric"):
-            __class__.pubsub_metric = Info(
-                'pubsub', 'Pub/sub configuration'
-            )
-
-        if not hasattr(__class__, "processing_metric"):
-            __class__.processing_metric = Counter(
-                'processing_count', 'Processing count', ["status"]
-            )
-
-        __class__.pubsub_metric.info({
-            "input_queue": input_queue,
-            "output_queue": output_queue,
-            "subscriber": subscriber,
-            "input_schema": input_schema.__name__,
-            "output_schema": output_schema.__name__,
-        })
-
-        super(ConsumerProducer, self).__init__(**params)
-
-        if input_schema == None:
-            raise RuntimeError("input_schema must be specified")
-
-        if output_schema == None:
-            raise RuntimeError("output_schema must be specified")
-
-        self.producer = self.client.create_producer(
-            topic=output_queue,
-            schema=JsonSchema(output_schema),
-            chunking_enabled=True,
-        )
-
-        self.consumer = self.client.subscribe(
-            input_queue, subscriber,
-            consumer_type=pulsar.ConsumerType.Shared,
-            schema=JsonSchema(input_schema),
-        )
-
-    def run(self):
-
-        __class__.state_metric.state('running')
-
-        while True:
-
-            msg = self.consumer.receive()
-
-            try:
-
-                with __class__.request_metric.time():
-                    resp = self.handle(msg)
-
-                # Acknowledge successful processing of the message
-                self.consumer.acknowledge(msg)
-
-                __class__.processing_metric.labels(status="success").inc()
-
-            except TooManyRequests:
-                self.consumer.negative_acknowledge(msg)
-                print("TooManyRequests: will retry")
-                __class__.processing_metric.labels(status="rate-limit").inc()
-                time.sleep(5)
-                continue
-
-            except Exception as e:
-
-                print("Exception:", e, flush=True)
-
-                # Message failed to be processed
-                self.consumer.negative_acknowledge(msg)
-
-                __class__.processing_metric.labels(status="error").inc()
-
-    def send(self, msg, properties={}):
-        self.producer.send(msg, properties)
-        __class__.output_metric.inc()
-
-    @staticmethod
-    def add_args(
-            parser, default_input_queue, default_subscriber,
-            default_output_queue,
-    ):
-
-        BaseProcessor.add_args(parser)
-
-        parser.add_argument(
-            '-i', '--input-queue',
-            default=default_input_queue,
-            help=f'Input queue (default: {default_input_queue})'
-        )
-
-        parser.add_argument(
-            '-s', '--subscriber',
-            default=default_subscriber,
-            help=f'Queue subscriber name (default: {default_subscriber})'
-        )
-
-        parser.add_argument(
-            '-o', '--output-queue',
-            default=default_output_queue,
-            help=f'Output queue (default: {default_output_queue})'
-        )
-

trustgraph-base-0.20.1/trustgraph/base_version.py

@@ -1 +0,0 @@
-__version__ = "0.20.1"