sentry-arroyo 2.31.0__tar.gz → 2.31.2__tar.gz

{sentry_arroyo-2.31.0/sentry_arroyo.egg-info → sentry_arroyo-2.31.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.31.0
+Version: 2.31.2
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.31.0 → sentry_arroyo-2.31.2}/arroyo/backends/kafka/consumer.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import logging
+import time
+from collections import defaultdict
 from concurrent.futures import Future
 from datetime import datetime
 from enum import Enum
@@ -21,6 +23,7 @@ from typing import (
     Tuple,
     Type,
     Union,
+    cast,
 )
 
 from confluent_kafka import (
@@ -748,6 +751,9 @@ class KafkaProducer(Producer[KafkaPayload]):
 # Type alias for the delivery callback function
 DeliveryCallback = Callable[[Optional[KafkaError], ConfluentMessage], None]
 
+# Interval between metric flushes (in seconds)
+METRICS_FREQUENCY_SEC = 1.0
+
 
 class ConfluentProducer(ConfluentKafkaProducer):  # type: ignore[misc]
     """
@@ -756,7 +762,10 @@ class ConfluentProducer(ConfluentKafkaProducer):  # type: ignore[misc]
 
     def __init__(self, configuration: Mapping[str, Any]) -> None:
         super().__init__(configuration)
+        self.producer_name = configuration.get("client.id") or None
         self.__metrics = get_metrics()
+        self.__produce_counters: MutableMapping[str, int] = defaultdict(int)
+        self.__reset_metrics()
 
     def __metrics_delivery_callback(
         self,
@@ -767,12 +776,8 @@ class ConfluentProducer(ConfluentKafkaProducer):  # type: ignore[misc]
             status = "error"
         else:
             status = "success"
-
-        self.__metrics.increment(
-            "arroyo.producer.produce_status",
-            1,
-            tags={"status": status},
-        )
+        self.__produce_counters[status] += 1
+        self.__throttled_record()
 
     def __delivery_callback(
         self,
@@ -792,3 +797,28 @@ class ConfluentProducer(ConfluentKafkaProducer):  # type: ignore[misc]
         user_callback = callback or on_delivery
         wrapped_callback = self.__delivery_callback(user_callback)
         super().produce(*args, on_delivery=wrapped_callback, **kwargs)
+
+    def __flush_metrics(self) -> None:
+        for status, count in self.__produce_counters.items():
+            tags = {"status": status}
+            if self.producer_name:
+                tags["producer_name"] = self.producer_name
+            self.__metrics.increment(
+                name="arroyo.producer.produce_status",
+                value=count,
+                tags=tags,
+            )
+        self.__reset_metrics()
+
+    def flush(self, timeout: float = -1) -> int:
+        # Kafka producer flush should flush metrics too
+        self.__flush_metrics()
+        return cast(int, super().flush(timeout))
+
+    def __reset_metrics(self) -> None:
+        self.__produce_counters.clear()
+        self.__last_record_time = time.time()
+
+    def __throttled_record(self) -> None:
+        if time.time() - self.__last_record_time > METRICS_FREQUENCY_SEC:
+            self.__flush_metrics()

{sentry_arroyo-2.31.0 → sentry_arroyo-2.31.2}/arroyo/processing/processor.py

@@ -146,9 +146,9 @@ class StreamProcessor(Generic[TStrategyPayload]):
         self.__processor_factory = processor_factory
         self.__metrics_buffer = MetricsBuffer()
 
-        self.__processing_strategy: Optional[
-            ProcessingStrategy[TStrategyPayload]
-        ] = None
+        self.__processing_strategy: Optional[ProcessingStrategy[TStrategyPayload]] = (
+            None
+        )
 
         self.__message: Optional[BrokerValue[TStrategyPayload]] = None
 

{sentry_arroyo-2.31.0 → sentry_arroyo-2.31.2/sentry_arroyo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.31.0
+Version: 2.31.2
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.31.0 → sentry_arroyo-2.31.2}/setup.py

@@ -10,7 +10,7 @@ def get_requirements() -> Sequence[str]:
 
 setup(
     name="sentry-arroyo",
-    version="2.31.0",
+    version="2.31.2",
     author="Sentry",
     author_email="oss@sentry.io",
     license="Apache-2.0",

{sentry_arroyo-2.31.0 → sentry_arroyo-2.31.2}/tests/backends/test_confluent_producer.py

@@ -24,11 +24,18 @@ class TestConfluentProducer:
 
     def test_metrics_callback_records_success(self) -> None:
         """Test that the metrics callback records success metric"""
-        producer = ConfluentProducer({"bootstrap.servers": "fake:9092"})
+        producer = ConfluentProducer(
+            {"bootstrap.servers": "fake:9092", "client.id": "test-producer-name"}
+        )
         mock_message = mock.Mock(spec=ConfluentMessage)
         producer._ConfluentProducer__metrics_delivery_callback(None, mock_message)
+        producer.flush()  # Flush buffered metrics
         assert (
-            Increment("arroyo.producer.produce_status", 1, {"status": "success"})
+            Increment(
+                "arroyo.producer.produce_status",
+                1,
+                {"status": "success", "producer_name": "test-producer-name"},
+            )
             in TestingMetricsBackend.calls
         )
 
@@ -38,6 +45,7 @@ class TestConfluentProducer:
         mock_error = mock.Mock(spec=KafkaError)
         mock_message = mock.Mock(spec=ConfluentMessage)
         producer._ConfluentProducer__metrics_delivery_callback(mock_error, mock_message)
+        producer.flush()  # Flush buffered metrics
         assert (
             Increment("arroyo.producer.produce_status", 1, {"status": "error"})
             in TestingMetricsBackend.calls
@@ -45,7 +53,9 @@ class TestConfluentProducer:
 
     def test_delivery_callback_wraps_user_callback(self) -> None:
         """Test that the delivery callback wrapper calls both metrics and user callbacks"""
-        producer = ConfluentProducer({"bootstrap.servers": "fake:9092"})
+        producer = ConfluentProducer(
+            {"bootstrap.servers": "fake:9092", "client.id": "test-producer-name"}
+        )
         user_callback_invoked = []
 
         def user_callback(
@@ -56,8 +66,13 @@ class TestConfluentProducer:
         wrapped = producer._ConfluentProducer__delivery_callback(user_callback)
         mock_message = mock.Mock(spec=ConfluentMessage)
         wrapped(None, mock_message)
+        producer.flush()  # Flush buffered metrics
         assert (
-            Increment("arroyo.producer.produce_status", 1, {"status": "success"})
+            Increment(
+                "arroyo.producer.produce_status",
+                1,
+                {"status": "success", "producer_name": "test-producer-name"},
+            )
             in TestingMetricsBackend.calls
         )
         assert len(user_callback_invoked) == 1