sentry-arroyo 2.31.1__py3-none-any.whl → 2.31.2__py3-none-any.whl

arroyo/backends/kafka/consumer.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import logging
+import time
+from collections import defaultdict
 from concurrent.futures import Future
 from datetime import datetime
 from enum import Enum
@@ -21,6 +23,7 @@ from typing import (
     Tuple,
     Type,
     Union,
+    cast,
 )
 
 from confluent_kafka import (
@@ -748,6 +751,9 @@ class KafkaProducer(Producer[KafkaPayload]):
 # Type alias for the delivery callback function
 DeliveryCallback = Callable[[Optional[KafkaError], ConfluentMessage], None]
 
+# Interval between metric flushes (in seconds)
+METRICS_FREQUENCY_SEC = 1.0
+
 
 class ConfluentProducer(ConfluentKafkaProducer):  # type: ignore[misc]
     """
@@ -756,8 +762,10 @@ class ConfluentProducer(ConfluentKafkaProducer):  # type: ignore[misc]
 
     def __init__(self, configuration: Mapping[str, Any]) -> None:
         super().__init__(configuration)
-        self.__metrics = get_metrics()
         self.producer_name = configuration.get("client.id") or None
+        self.__metrics = get_metrics()
+        self.__produce_counters: MutableMapping[str, int] = defaultdict(int)
+        self.__reset_metrics()
 
     def __metrics_delivery_callback(
         self,
@@ -768,16 +776,8 @@ class ConfluentProducer(ConfluentKafkaProducer):  # type: ignore[misc]
             status = "error"
         else:
             status = "success"
-
-        tags = {"status": status}
-        if self.producer_name:
-            tags["producer_name"] = self.producer_name
-
-        self.__metrics.increment(
-            "arroyo.producer.produce_status",
-            1,
-            tags=tags,
-        )
+        self.__produce_counters[status] += 1
+        self.__throttled_record()
 
     def __delivery_callback(
         self,
@@ -797,3 +797,28 @@ class ConfluentProducer(ConfluentKafkaProducer):  # type: ignore[misc]
         user_callback = callback or on_delivery
         wrapped_callback = self.__delivery_callback(user_callback)
         super().produce(*args, on_delivery=wrapped_callback, **kwargs)
+
+    def __flush_metrics(self) -> None:
+        for status, count in self.__produce_counters.items():
+            tags = {"status": status}
+            if self.producer_name:
+                tags["producer_name"] = self.producer_name
+            self.__metrics.increment(
+                name="arroyo.producer.produce_status",
+                value=count,
+                tags=tags,
+            )
+        self.__reset_metrics()
+
+    def flush(self, timeout: float = -1) -> int:
+        # Kafka producer flush should flush metrics too
+        self.__flush_metrics()
+        return cast(int, super().flush(timeout))
+
+    def __reset_metrics(self) -> None:
+        self.__produce_counters.clear()
+        self.__last_record_time = time.time()
+
+    def __throttled_record(self) -> None:
+        if time.time() - self.__last_record_time > METRICS_FREQUENCY_SEC:
+            self.__flush_metrics()

arroyo/processing/processor.py

@@ -146,9 +146,9 @@ class StreamProcessor(Generic[TStrategyPayload]):
         self.__processor_factory = processor_factory
         self.__metrics_buffer = MetricsBuffer()
 
-        self.__processing_strategy: Optional[
-            ProcessingStrategy[TStrategyPayload]
-        ] = None
+        self.__processing_strategy: Optional[ProcessingStrategy[TStrategyPayload]] = (
+            None
+        )
 
         self.__message: Optional[BrokerValue[TStrategyPayload]] = None
 

{sentry_arroyo-2.31.1.dist-info → sentry_arroyo-2.31.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.31.1
+Version: 2.31.2
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.31.1.dist-info → sentry_arroyo-2.31.2.dist-info}/RECORD

@@ -9,14 +9,14 @@ arroyo/backends/abstract.py,sha256=NQ5xG9rjchSUW8URl1WtSYSzMFtwRKB6wtJhWmDyR9E,9
 arroyo/backends/kafka/__init__.py,sha256=xgf-AqHbQkJsh73YokO2uoyyHfZf8XwUp6BULtM8stI,445
 arroyo/backends/kafka/commit.py,sha256=LPsjvX5PPXR62DT6sa5GuSF78qk9F_L--Fz4kw7-m-s,3060
 arroyo/backends/kafka/configuration.py,sha256=g1Y-vdu3OT9pqWuYRp9fw29Nmm1KBBExQLr3VhDsp90,8950
-arroyo/backends/kafka/consumer.py,sha256=QrhgcTiJbwZ3rf_nOn9PQL9n8L9chyXmtTGPRRiBxoY,30130
+arroyo/backends/kafka/consumer.py,sha256=hABMHwTFm0IMziSCTxDQkepvATjcKCjDB0l_NFPypKs,31093
 arroyo/backends/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/backends/local/backend.py,sha256=7odjCnzoGgdo8JHLgG1ntaXa-ZR9GteGkquiA2WAWmM,13880
 arroyo/backends/local/storages/__init__.py,sha256=AGYujdAAcn3osoj9jq84IzTywYbkIDv9wRg2rLhLXeg,104
 arroyo/backends/local/storages/abstract.py,sha256=1qVQp6roxHkK6XT2aklZyZk1qq7RzcPN6Db_CA5--kg,2901
 arroyo/backends/local/storages/memory.py,sha256=AoKDsVZzBXkOJyWArKWp3vfGfU9xLlKFXE9gsJiMIzQ,2613
 arroyo/processing/__init__.py,sha256=vZVg0wJvJfoVzlzGvnL59bT6YNIRJNQ5t7oU045Qbk4,87
-arroyo/processing/processor.py,sha256=WRQUnakUWctWlEcBW1s_f-xbd3dPOJ1UCOsvVYlfvLQ,21321
+arroyo/processing/processor.py,sha256=PeuCnnwp2Ehxkl3Wj_81ICKbWe45OJya1rByYcYJNlA,21323
 arroyo/processing/strategies/__init__.py,sha256=EU_JMb54eOxMxaC5mIFpI-sAF-X2ZScbE8czBZ7bQkY,1106
 arroyo/processing/strategies/abstract.py,sha256=nu7juEz_aQmQIH35Z8u--FBuLjkK8_LQ1hIG2xpw9AA,4808
 arroyo/processing/strategies/batching.py,sha256=s89xC6lQpBseEaApu1iNTipXGKeO95OMwinj2VBKn9s,4778
@@ -46,11 +46,11 @@ examples/transform_and_produce/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 examples/transform_and_produce/batched.py,sha256=st2R6qTneAtV0JFbKP30Ti3sJDYj8Jkbmta9JckKdZU,2636
 examples/transform_and_produce/script.py,sha256=8kSMIjQNqGYEVyE0PvrfJh-a_UYCrJSstTp_De7kyyg,2306
 examples/transform_and_produce/simple.py,sha256=H7xqxItjl4tx34wVW5dy6mB9G39QucAtxkJSBzVmjgA,1637
-sentry_arroyo-2.31.1.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
+sentry_arroyo-2.31.2.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
 tests/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/backends/mixins.py,sha256=sfNyE0VTeiD3GHOnBYl-9urvPuURI2G1BWke0cz7Dvc,20445
 tests/backends/test_commit.py,sha256=iTHfK1qsBxim0XwxgMvNNSMqDUMEHoYkYBDcgxGBFbs,831
-tests/backends/test_confluent_producer.py,sha256=zMCRqXdrWSgkwIspZJ7uhzpkQIFZKNbQeMAG1BeQPms,2993
+tests/backends/test_confluent_producer.py,sha256=KWqgvjDvqAdd0HxngdWKsUJaV7Hl1L5vAVQhBYlHeHU,3146
 tests/backends/test_kafka.py,sha256=6W6EA41X-ECTfcOeivhQxURnmV2Y0fYy-UzDCnJgDsU,12830
 tests/backends/test_kafka_producer.py,sha256=LpwkqnstcCDxemlKZ0FpzNKrP-1UuXXY15P7P-spjhE,3912
 tests/backends/test_local.py,sha256=Mfd4DFuWVSVtl1GomQ6TIoWuJNcAliKqKU0BShPlEMY,3363
@@ -74,7 +74,7 @@ tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/utils/test_concurrent.py,sha256=Gwdzym2UZ1HO3rhOSGmzxImWcLFygY8P7MXHT3Q0xTE,455
 tests/utils/test_metrics.py,sha256=bI0EtGgPokMQyEqX58i0-8zvLfxRP2nWaWr2wLMaJ_o,917
 tests/utils/test_retries.py,sha256=AxJLkXWeL9AjHv_p1n0pe8CXXJp24ZQIuYBHfNcmiz4,3075
-sentry_arroyo-2.31.1.dist-info/METADATA,sha256=JREC1Nm6b51e0nmz9ANO0N86Qh2th772Ri8t-I5FXJc,2208
-sentry_arroyo-2.31.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sentry_arroyo-2.31.1.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
-sentry_arroyo-2.31.1.dist-info/RECORD,,
+sentry_arroyo-2.31.2.dist-info/METADATA,sha256=3pXtE_zg-qbvUHSaMkpixJnBkFHmdNI-2Ln8YBlZkX0,2208
+sentry_arroyo-2.31.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sentry_arroyo-2.31.2.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
+sentry_arroyo-2.31.2.dist-info/RECORD,,

tests/backends/test_confluent_producer.py

@@ -29,6 +29,7 @@ class TestConfluentProducer:
         )
         mock_message = mock.Mock(spec=ConfluentMessage)
         producer._ConfluentProducer__metrics_delivery_callback(None, mock_message)
+        producer.flush()  # Flush buffered metrics
         assert (
             Increment(
                 "arroyo.producer.produce_status",
@@ -44,6 +45,7 @@ class TestConfluentProducer:
         mock_error = mock.Mock(spec=KafkaError)
         mock_message = mock.Mock(spec=ConfluentMessage)
         producer._ConfluentProducer__metrics_delivery_callback(mock_error, mock_message)
+        producer.flush()  # Flush buffered metrics
         assert (
             Increment("arroyo.producer.produce_status", 1, {"status": "error"})
             in TestingMetricsBackend.calls
@@ -64,6 +66,7 @@ class TestConfluentProducer:
         wrapped = producer._ConfluentProducer__delivery_callback(user_callback)
         mock_message = mock.Mock(spec=ConfluentMessage)
         wrapped(None, mock_message)
+        producer.flush()  # Flush buffered metrics
         assert (
             Increment(
                 "arroyo.producer.produce_status",