trustgraph-base 2.8.2__tar.gz → 2.8.3__tar.gz

{trustgraph_base-2.8.2 → trustgraph_base-2.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-base
-Version: 2.8.2
+Version: 2.8.3
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

{trustgraph_base-2.8.2 → trustgraph_base-2.8.3}/trustgraph/base/async_processor.py

@@ -51,10 +51,12 @@ class AsyncProcessor:
         self.receiver_pool = ReceiverPool(
             backend=self.async_backend,
             concurrency=self.concurrency,
+            processor_id=self.id,
         )
 
         self.sender_pool = SenderPool(
             backend=self.async_backend,
+            processor_id=self.id,
         )
 
         self.config_push_queue = params.get(

{trustgraph_base-2.8.2 → trustgraph_base-2.8.3}/trustgraph/base/metrics.py

@@ -13,13 +13,11 @@ class ConsumerMetrics:
 
         self.processor = processor
         self.consumer = consumer
-        self.workspace = workspace
-        self.flow = flow
 
         if not hasattr(__class__, "state_metric"):
             __class__.state_metric = Enum(
                 'consumer_state', 'Consumer state',
-                ["processor", "workspace", "flow", "consumer"],
+                ["processor", "consumer"],
                 states=['stopped', 'running']
             )
 
@@ -32,31 +30,46 @@ class ConsumerMetrics:
         if not hasattr(__class__, "processing_metric"):
             __class__.processing_metric = Counter(
                 'processing_count', 'Processing count',
-                ["processor", "workspace", "flow", "consumer", "status"],
+                ["processor", "consumer", "status"],
             )
 
         if not hasattr(__class__, "rate_limit_metric"):
             __class__.rate_limit_metric = Counter(
                 'rate_limit_count', 'Rate limit event count',
-                ["processor", "workspace", "flow", "consumer"],
+                ["processor", "consumer"],
             )
 
+        __class__.request_metric.labels(
+            processor=self.processor, consumer=self.consumer,
+        )
+        __class__.processing_metric.labels(
+            processor=self.processor, consumer=self.consumer,
+            status="ok",
+        )
+        __class__.processing_metric.labels(
+            processor=self.processor, consumer=self.consumer,
+            status="error",
+        )
+        __class__.rate_limit_metric.labels(
+            processor=self.processor, consumer=self.consumer,
+        )
+
     def process(self, status: str) -> None:
         __class__.processing_metric.labels(
-            processor=self.processor, workspace=self.workspace,
-            flow=self.flow, consumer=self.consumer, status=status,
+            processor=self.processor,
+            consumer=self.consumer, status=status,
         ).inc()
 
     def rate_limit(self) -> None:
         __class__.rate_limit_metric.labels(
-            processor=self.processor, workspace=self.workspace,
-            flow=self.flow, consumer=self.consumer,
+            processor=self.processor,
+            consumer=self.consumer,
         ).inc()
 
     def state(self, state: str) -> None:
         __class__.state_metric.labels(
-            processor=self.processor, workspace=self.workspace,
-            flow=self.flow, consumer=self.consumer,
+            processor=self.processor,
+            consumer=self.consumer,
         ).state(state)
 
     def record_time(self) -> Any:
@@ -72,19 +85,21 @@ class ProducerMetrics:
 
         self.processor = processor
         self.producer = producer
-        self.workspace = workspace
-        self.flow = flow
 
         if not hasattr(__class__, "producer_metric"):
             __class__.producer_metric = Counter(
                 'producer_count', 'Output items produced',
-                ["processor", "workspace", "flow", "producer"],
+                ["processor", "producer"],
             )
 
+        __class__.producer_metric.labels(
+            processor=self.processor, producer=self.producer,
+        )
+
     def inc(self) -> None:
         __class__.producer_metric.labels(
-            processor=self.processor, workspace=self.workspace,
-            flow=self.flow, producer=self.producer,
+            processor=self.processor,
+            producer=self.producer,
         ).inc()
 
 class ProcessorMetrics:
@@ -111,42 +126,40 @@ class SubscriberMetrics:
 
         self.processor = processor
         self.subscriber = subscriber
-        self.workspace = workspace
-        self.flow = flow
 
         if not hasattr(__class__, "state_metric"):
             __class__.state_metric = Enum(
                 'subscriber_state', 'Subscriber state',
-                ["processor", "workspace", "flow", "subscriber"],
+                ["processor", "subscriber"],
                 states=['stopped', 'running']
             )
 
         if not hasattr(__class__, "received_metric"):
             __class__.received_metric = Counter(
                 'received_count', 'Received count',
-                ["processor", "workspace", "flow", "subscriber"],
+                ["processor", "subscriber"],
             )
 
         if not hasattr(__class__, "dropped_metric"):
             __class__.dropped_metric = Counter(
                 'dropped_count', 'Dropped messages count',
-                ["processor", "workspace", "flow", "subscriber"],
+                ["processor", "subscriber"],
             )
 
     def received(self) -> None:
         __class__.received_metric.labels(
-            processor=self.processor, workspace=self.workspace,
-            flow=self.flow, subscriber=self.subscriber,
+            processor=self.processor,
+            subscriber=self.subscriber,
         ).inc()
 
     def state(self, state: str) -> None:
         __class__.state_metric.labels(
-            processor=self.processor, workspace=self.workspace,
-            flow=self.flow, subscriber=self.subscriber,
+            processor=self.processor,
+            subscriber=self.subscriber,
         ).state(state)
 
     def dropped(self, state: str) -> None:
         __class__.dropped_metric.labels(
-            processor=self.processor, workspace=self.workspace,
-            flow=self.flow, subscriber=self.subscriber,
+            processor=self.processor,
+            subscriber=self.subscriber,
         ).inc()

{trustgraph_base-2.8.2 → trustgraph_base-2.8.3}/trustgraph/base/receiver_pool.py

@@ -26,6 +26,7 @@ class ConsumerRegistration:
     backend_consumer: AsyncBackendConsumer
     receiver_task: asyncio.Task
     pool: 'ReceiverPool'
+    metrics: Any = None
 
     async def unregister(self):
         await self.pool.remove_consumer(self)
@@ -58,9 +59,13 @@ class ReceiverPool:
         await pool.stop()
     """
 
-    def __init__(self, backend: AsyncPubSubBackend, concurrency: int = 1):
+    def __init__(
+        self, backend: AsyncPubSubBackend, concurrency: int = 1,
+        processor_id: str | None = None,
+    ):
         self.backend = backend
         self.concurrency = concurrency
+        self.processor_id = processor_id
         self.work_queue = asyncio.Queue(maxsize=concurrency)
         self.registrations: list[ConsumerRegistration] = []
         self.worker_tasks: list[asyncio.Task] = []
@@ -93,6 +98,11 @@ class ReceiverPool:
                 await reg.receiver_task
             except BaseException:
                 pass
+
+        for reg in self.registrations:
+            if reg.metrics:
+                reg.metrics.state("stopped")
+
         self.registrations.clear()
 
         if not self.work_queue.empty():
@@ -136,8 +146,18 @@ class ReceiverPool:
             initial_position=initial_position,
         )
 
+        metrics = None
+        if self.processor_id:
+            from .metrics import ConsumerMetrics
+            metrics = ConsumerMetrics(
+                processor=self.processor_id, consumer=topic,
+            )
+            metrics.state("running")
+
+        instrumented = self._instrumented_handler(handler, metrics)
+
         receiver_task = asyncio.create_task(
-            self._receiver_loop(backend_consumer, handler),
+            self._receiver_loop(backend_consumer, instrumented),
             name=f"receiver-{topic}",
         )
 
@@ -147,12 +167,32 @@ class ReceiverPool:
             backend_consumer=backend_consumer,
             receiver_task=receiver_task,
             pool=self,
+            metrics=metrics,
         )
         self.registrations.append(reg)
 
         logger.info(f"Added consumer: {topic}")
         return reg
 
+    def _instrumented_handler(
+        self,
+        handler: Callable[..., Awaitable[None]],
+        metrics,
+    ) -> Callable[..., Awaitable[None]]:
+        if metrics is None:
+            return handler
+
+        async def wrapper(message):
+            with metrics.record_time():
+                try:
+                    await handler(message)
+                    metrics.process("ok")
+                except Exception:
+                    metrics.process("error")
+                    raise
+
+        return wrapper
+
     async def remove_consumer(self, reg: ConsumerRegistration):
         try:
             await reg.backend_consumer.close()
@@ -168,6 +208,9 @@ class ReceiverPool:
         if reg in self.registrations:
             self.registrations.remove(reg)
 
+        if reg.metrics:
+            reg.metrics.state("stopped")
+
         logger.info(f"Removed consumer: {reg.topic}")
 
     async def _receiver_loop(

{trustgraph_base-2.8.2 → trustgraph_base-2.8.3}/trustgraph/base/sender_pool.py

@@ -31,11 +31,13 @@ class ProducerHandle:
     def __init__(
         self, topic: str, backend_producer: AsyncBackendProducer,
         send_queue: asyncio.Queue, pool: 'SenderPool',
+        metrics=None,
     ):
         self.topic = topic
         self.backend_producer = backend_producer
         self.send_queue = send_queue
         self.pool = pool
+        self.metrics = metrics
 
     async def send(
         self, message: Any, properties: dict = {}, wait: bool = False,
@@ -49,6 +51,9 @@ class ProducerHandle:
             SendItem(message=message, properties=properties, future=future)
         )
 
+        if self.metrics:
+            self.metrics.inc()
+
         if future is not None:
             await future
 
@@ -75,9 +80,11 @@ class SenderPool:
 
     def __init__(
         self, backend: AsyncPubSubBackend, send_queue_size: int = 64,
+        processor_id: str | None = None,
     ):
         self.backend = backend
         self.send_queue_size = send_queue_size
+        self.processor_id = processor_id
         self.handles: list[ProducerHandle] = []
         self.sender_task: asyncio.Task | None = None
         self.running = False
@@ -130,11 +137,19 @@ class SenderPool:
 
         send_queue = asyncio.Queue(maxsize=self.send_queue_size)
 
+        metrics = None
+        if self.processor_id:
+            from .metrics import ProducerMetrics
+            metrics = ProducerMetrics(
+                processor=self.processor_id, producer=topic,
+            )
+
         handle = ProducerHandle(
             topic=topic,
             backend_producer=backend_producer,
             send_queue=send_queue,
             pool=self,
+            metrics=metrics,
         )
         self.handles.append(handle)
 

{trustgraph_base-2.8.2 → trustgraph_base-2.8.3}/trustgraph/base/tool_service.py

@@ -50,7 +50,7 @@ class ToolService(FlowProcessor):
         if not hasattr(__class__, "tool_invocation_metric"):
             __class__.tool_invocation_metric = Counter(
                 'tool_invocation_count', 'Tool invocation count',
-                ["processor", "workspace", "flow", "tool"],
+                ["processor", "tool"],
             )
 
     async def on_request(self, msg, consumer, flow):
@@ -89,8 +89,7 @@ class ToolService(FlowProcessor):
                 )
 
             __class__.tool_invocation_metric.labels(
-                processor=self.id, workspace=flow.workspace,
-                flow=flow.name, tool=request.name,
+                processor=self.id, tool=request.name,
             ).inc()
 
         except TooManyRequests as e:

trustgraph_base-2.8.3/trustgraph/base_version.py

@@ -0,0 +1 @@
+__version__ = "2.8.3"

{trustgraph_base-2.8.2 → trustgraph_base-2.8.3}/trustgraph_base.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-base
-Version: 2.8.2
+Version: 2.8.3
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

trustgraph_base-2.8.2/trustgraph/base_version.py

@@ -1 +0,0 @@
-__version__ = "2.8.2"