sentry-arroyo 2.24.0__tar.gz → 2.26.0__tar.gz

{sentry_arroyo-2.24.0/sentry_arroyo.egg-info → sentry_arroyo-2.26.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.24.0
+Version: 2.26.0
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.24.0 → sentry_arroyo-2.26.0}/arroyo/backends/kafka/__init__.py

@@ -1,9 +1,14 @@
-from .configuration import build_kafka_configuration, build_kafka_consumer_configuration
+from .configuration import (
+    build_kafka_configuration,
+    build_kafka_consumer_configuration,
+    build_kafka_producer_configuration,
+)
 from .consumer import KafkaConsumer, KafkaPayload, KafkaProducer
 
 __all__ = [
     "build_kafka_configuration",
     "build_kafka_consumer_configuration",
+    "build_kafka_producer_configuration",
     "KafkaConsumer",
     "KafkaPayload",
     "KafkaProducer",

{sentry_arroyo-2.24.0 → sentry_arroyo-2.26.0}/arroyo/backends/kafka/configuration.py

@@ -49,6 +49,77 @@ def stats_callback(stats_json: str) -> None:
     )
 
 
+def producer_stats_callback(stats_json: str) -> None:
+    stats = json.loads(stats_json)
+    metrics = get_metrics()
+
+    # Extract broker-level int_latency metrics
+    brokers = stats.get("brokers", {})
+    for broker_id, broker_stats in brokers.items():
+        int_latency = broker_stats.get("int_latency", {})
+        if int_latency:
+            p99_latency_ms = int_latency.get("p99", 0) / 1000.0
+            metrics.timing(
+                "arroyo.producer.librdkafka.p99_int_latency",
+                p99_latency_ms,
+                tags={"broker_id": str(broker_id)},
+            )
+            avg_latency_ms = int_latency.get("avg", 0) / 1000.0
+            metrics.timing(
+                "arroyo.producer.librdkafka.avg_int_latency",
+                avg_latency_ms,
+                tags={"broker_id": str(broker_id)},
+            )
+
+        outbuf_latency = broker_stats.get("outbuf_latency", {})
+        if outbuf_latency:
+            p99_latency_ms = outbuf_latency.get("p99", 0) / 1000.0
+            metrics.timing(
+                "arroyo.producer.librdkafka.p99_outbuf_latency",
+                p99_latency_ms,
+                tags={"broker_id": str(broker_id)},
+            )
+            avg_latency_ms = outbuf_latency.get("avg", 0) / 1000.0
+            metrics.timing(
+                "arroyo.producer.librdkafka.avg_outbuf_latency",
+                avg_latency_ms,
+                tags={"broker_id": str(broker_id)},
+            )
+
+        rtt = broker_stats.get("rtt", {})
+        if rtt:
+            p99_rtt_ms = rtt.get("p99", 0) / 1000.0
+            metrics.timing(
+                "arroyo.producer.librdkafka.p99_rtt",
+                p99_rtt_ms,
+                tags={"broker_id": str(broker_id)},
+            )
+            avg_rtt_ms = rtt.get("avg", 0) / 1000.0
+            metrics.timing(
+                "arroyo.producer.librdkafka.avg_rtt",
+                avg_rtt_ms,
+                tags={"broker_id": str(broker_id)},
+            )
+
+
+def build_kafka_producer_configuration(
+    default_config: Mapping[str, Any],
+    bootstrap_servers: Optional[Sequence[str]] = None,
+    override_params: Optional[Mapping[str, Any]] = None,
+) -> KafkaBrokerConfig:
+    broker_config = build_kafka_configuration(
+        default_config, bootstrap_servers, override_params
+    )
+
+    broker_config.update(
+        {
+            "statistics.interval.ms": STATS_COLLECTION_FREQ_MS,
+            "stats_cb": producer_stats_callback,
+        }
+    )
+    return broker_config
+
+
 def build_kafka_consumer_configuration(
     default_config: Mapping[str, Any],
     group_id: str,

{sentry_arroyo-2.24.0 → sentry_arroyo-2.26.0}/arroyo/processing/processor.py

@@ -29,7 +29,7 @@ from arroyo.processing.strategies.abstract import (
 )
 from arroyo.types import BrokerValue, Message, Partition, Topic, TStrategyPayload
 from arroyo.utils.logging import handle_internal_error
-from arroyo.utils.metrics import get_metrics
+from arroyo.utils.metrics import get_consumer_metrics
 
 logger = logging.getLogger(__name__)
 
@@ -90,7 +90,7 @@ ConsumerCounter = Literal[
 
 class MetricsBuffer:
     def __init__(self) -> None:
-        self.metrics = get_metrics()
+        self.metrics = get_consumer_metrics()
         self.__timers: MutableMapping[ConsumerTiming, float] = defaultdict(float)
         self.__counters: MutableMapping[ConsumerCounter, int] = defaultdict(int)
         self.__reset()
@@ -139,6 +139,7 @@ class StreamProcessor(Generic[TStrategyPayload]):
         commit_policy: CommitPolicy = ONCE_PER_SECOND,
         dlq_policy: Optional[DlqPolicy[TStrategyPayload]] = None,
         join_timeout: Optional[float] = None,
+        shutdown_strategy_before_consumer: bool = False,
     ) -> None:
         self.__consumer = consumer
         self.__processor_factory = processor_factory
@@ -158,6 +159,7 @@ class StreamProcessor(Generic[TStrategyPayload]):
         self.__commit_policy_state = commit_policy.get_state_machine()
         self.__join_timeout = join_timeout
         self.__shutdown_requested = False
+        self.__shutdown_strategy_before_consumer = shutdown_strategy_before_consumer
 
         # Buffers messages for DLQ. Messages are added when they are submitted for processing and
         # removed once the commit callback is fired as they are guaranteed to be valid at that point.
@@ -170,49 +172,7 @@ class StreamProcessor(Generic[TStrategyPayload]):
         )
 
         def _close_strategy() -> None:
-            start_close = time.time()
-
-            if self.__processing_strategy is None:
-                # Partitions are revoked when the consumer is shutting down, at
-                # which point we already have closed the consumer.
-                return
-
-            logger.info("Closing %r...", self.__processing_strategy)
-            self.__processing_strategy.close()
-
-            logger.info("Waiting for %r to exit...", self.__processing_strategy)
-
-            while True:
-                start_join = time.time()
-
-                try:
-                    self.__processing_strategy.join(self.__join_timeout)
-                    self.__metrics_buffer.incr_timing(
-                        "arroyo.consumer.join.time", time.time() - start_join
-                    )
-                    break
-                except InvalidMessage as e:
-                    self.__metrics_buffer.incr_timing(
-                        "arroyo.consumer.join.time", time.time() - start_join
-                    )
-                    self._handle_invalid_message(e)
-
-            logger.info(
-                "%r exited successfully, releasing assignment.",
-                self.__processing_strategy,
-            )
-            self.__processing_strategy = None
-            self.__message = None  # avoid leaking buffered messages across assignments
-            self.__is_paused = False
-            self._clear_backpressure()
-
-            value = time.time() - start_close
-
-            self.__metrics_buffer.metrics.timing(
-                "arroyo.consumer.run.close_strategy", value
-            )
-
-            self.__metrics_buffer.incr_timing("arroyo.consumer.shutdown.time", value)
+            self._close_processing_strategy()
 
         def _create_strategy(partitions: Mapping[Partition, int]) -> None:
             start_create = time.time()
@@ -235,8 +195,10 @@ class StreamProcessor(Generic[TStrategyPayload]):
         def on_partitions_assigned(partitions: Mapping[Partition, int]) -> None:
             logger.info("New partitions assigned: %r", partitions)
             logger.info("Member id: %r", self.__consumer.member_id)
+            self.__metrics_buffer.metrics.consumer_member_id = self.__consumer.member_id
+
             self.__metrics_buffer.metrics.increment(
-                "arroyo.consumer.partitions_assigned.count", len(partitions), tags={"consumer_member_id": self.__consumer.member_id}
+                "arroyo.consumer.partitions_assigned.count", len(partitions)
             )
 
             current_partitions = dict(self.__consumer.tell())
@@ -244,6 +206,7 @@ class StreamProcessor(Generic[TStrategyPayload]):
 
             if self.__dlq_policy:
                 self.__dlq_policy.reset_dlq_limits(current_partitions)
+
             if current_partitions:
                 if self.__processing_strategy is not None:
                     # TODO: for cooperative-sticky rebalancing this can happen
@@ -262,7 +225,7 @@ class StreamProcessor(Generic[TStrategyPayload]):
             logger.info("Partitions to revoke: %r", partitions)
 
             self.__metrics_buffer.metrics.increment(
-                "arroyo.consumer.partitions_revoked.count", len(partitions), tags={"consumer_member_id": self.__consumer.member_id}
+                "arroyo.consumer.partitions_revoked.count", len(partitions)
             )
 
             if partitions:
@@ -299,6 +262,48 @@ class StreamProcessor(Generic[TStrategyPayload]):
             [topic], on_assign=on_partitions_assigned, on_revoke=on_partitions_revoked
         )
 
+    def _close_processing_strategy(self) -> None:
+        """Close the processing strategy and wait for it to exit."""
+        start_close = time.time()
+
+        if self.__processing_strategy is None:
+            # Partitions are revoked when the consumer is shutting down, at
+            # which point we already have closed the consumer.
+            return
+
+        logger.info("Closing %r...", self.__processing_strategy)
+        logger.info("Member id: %r", self.__consumer.member_id)
+        self.__processing_strategy.close()
+
+        logger.info("Waiting for %r to exit...", self.__processing_strategy)
+
+        while True:
+            start_join = time.time()
+
+            try:
+                self.__processing_strategy.join(self.__join_timeout)
+                self.__metrics_buffer.incr_timing(
+                    "arroyo.consumer.join.time", time.time() - start_join
+                )
+                break
+            except InvalidMessage as e:
+                self.__metrics_buffer.incr_timing(
+                    "arroyo.consumer.join.time", time.time() - start_join
+                )
+                self._handle_invalid_message(e)
+
+        logger.info("%r exited successfully", self.__processing_strategy)
+        self.__processing_strategy = None
+        self.__message = None
+        self.__is_paused = False
+        self._clear_backpressure()
+
+        value = time.time() - start_close
+        self.__metrics_buffer.metrics.timing(
+            "arroyo.consumer.run.close_strategy", value
+        )
+        self.__metrics_buffer.incr_timing("arroyo.consumer.shutdown.time", value)
+
     def __commit(self, offsets: Mapping[Partition, int], force: bool = False) -> None:
         """
         If force is passed, commit immediately and do not throttle. This should
@@ -515,6 +520,13 @@ class StreamProcessor(Generic[TStrategyPayload]):
         self.__shutdown_requested = True
 
     def _shutdown(self) -> None:
+        # If shutdown_strategy_before_consumer is set, work around an issue
+        # where rdkafka would revoke our partition, but then also immediately
+        # revoke our member ID as well, causing join() of the CommitStrategy
+        # (that is running in the partition revocation callback) to crash.
+        if self.__shutdown_strategy_before_consumer:
+            self._close_processing_strategy()
+
         # close the consumer
         logger.info("Stopping consumer")
         self.__metrics_buffer.flush()

{sentry_arroyo-2.24.0 → sentry_arroyo-2.26.0}/arroyo/utils/metricDefs.json

@@ -1 +1 @@
-{"arroyo.strategies.run_task_with_multiprocessing.batch.size.msg": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.size.msg", "type": "Time", "description": "Number of messages in a multiprocessing batch"}, "arroyo.strategies.run_task_with_multiprocessing.batch.size.bytes": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.size.bytes", "type": "Time", "description": "Number of bytes in a multiprocessing batch"}, "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.msg": {"name": "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.msg", "type": "Time", "description": "Number of messages in a multiprocessing batch after the message transformation"}, "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.bytes": {"name": "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.bytes", "type": "Time", "description": "Number of bytes in a multiprocessing batch after the message transformation"}, "arroyo.consumer.run.count": {"name": "arroyo.consumer.run.count", "type": "Counter", "description": "Number of times the consumer is spinning"}, "arroyo.consumer.invalid_message.count": {"name": "arroyo.consumer.invalid_message.count", "type": "Counter", "description": "Number of times the consumer encountered an invalid message."}, "arroyo.strategies.reduce.batch_time": {"name": "arroyo.strategies.reduce.batch_time", "type": "Time", "description": "How long it took the Reduce step to fill up a batch"}, "arroyo.strategies.run_task_with_multiprocessing.batch.backpressure": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.backpressure", "type": "Counter", "description": "Incremented when a strategy after multiprocessing applies\nbackpressure to multiprocessing. May be a reason why CPU cannot be\nsaturated."}, "arroyo.strategies.run_task_with_multiprocessing.batch.input.overflow": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.input.overflow", "type": "Counter", "description": "Incremented when multiprocessing cannot fill the input batch\nbecause not enough memory was allocated. This results in batches smaller\nthan configured. Increase `input_block_size` to fix."}, "arroyo.strategies.run_task_with_multiprocessing.batch.output.overflow": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.output.overflow", "type": "Counter", "description": "Incremented when multiprocessing cannot pull results in batches\nequal to the input batch size, because not enough memory was allocated.\nThis can be devastating for throughput. Increase `output_block_size` to\nfix."}, "arroyo.strategies.run_task_with_multiprocessing.batch.input.resize": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.input.resize", "type": "Counter", "description": "Arroyo has decided to re-allocate a block in order to combat input\nbuffer overflow. This behavior can be disabled by explicitly setting\n`input_block_size` to a not-None value in `RunTaskWithMultiprocessing`."}, "arroyo.strategies.run_task_with_multiprocessing.batch.output.resize": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.output.resize", "type": "Counter", "description": "Arroyo has decided to re-allocate a block in order to combat output\nbuffer overflow. This behavior can be disabled by explicitly setting\n`output_block_size` to a not-None value in `RunTaskWithMultiprocessing`."}, "arroyo.strategies.run_task_with_multiprocessing.batches_in_progress": {"name": "arroyo.strategies.run_task_with_multiprocessing.batches_in_progress", "type": "Gauge", "description": "How many batches are being processed in parallel by multiprocessing."}, "arroyo.strategies.run_task_with_multiprocessing.processes": {"name": "arroyo.strategies.run_task_with_multiprocessing.processes", "type": "Counter", "description": "A subprocess by multiprocessing unexpectedly died.\n\"sigchld.detected\",\nGauge: Shows how many processes the multiprocessing strategy is\nconfigured with."}, "arroyo.strategies.run_task_with_multiprocessing.pool.create": {"name": "arroyo.strategies.run_task_with_multiprocessing.pool.create", "type": "Counter", "description": "Incremented when the multiprocessing pool is created (or re-created)."}, "arroyo.consumer.poll.time": {"name": "arroyo.consumer.poll.time", "type": "Time", "description": "(unitless) spent polling librdkafka for new messages."}, "arroyo.consumer.processing.time": {"name": "arroyo.consumer.processing.time", "type": "Time", "description": "(unitless) spent in strategies (blocking in strategy.submit or\nstrategy.poll)"}, "arroyo.consumer.backpressure.time": {"name": "arroyo.consumer.backpressure.time", "type": "Time", "description": "(unitless) spent pausing the consumer due to backpressure (MessageRejected)"}, "arroyo.consumer.dlq.time": {"name": "arroyo.consumer.dlq.time", "type": "Time", "description": "(unitless) spent in handling `InvalidMessage` exceptions and sending\nmessages to the the DLQ."}, "arroyo.consumer.join.time": {"name": "arroyo.consumer.join.time", "type": "Time", "description": "(unitless) spent in waiting for the strategy to exit, such as during\nshutdown or rebalancing."}, "arroyo.consumer.callback.time": {"name": "arroyo.consumer.callback.time", "type": "Time", "description": "(unitless) spent in librdkafka callbacks. This metric's timings\noverlap other timings, and might spike at the same time."}, "arroyo.consumer.shutdown.time": {"name": "arroyo.consumer.shutdown.time", "type": "Time", "description": "(unitless) spent in shutting down the consumer. This metric's\ntimings overlap other timings, and might spike at the same time."}, "arroyo.consumer.run.callback": {"name": "arroyo.consumer.run.callback", "type": "Time", "description": "A regular duration metric where each datapoint is measuring the time it\ntook to execute a single callback. This metric is distinct from the\narroyo.consumer.*.time metrics as it does not attempt to accumulate time\nspent per second in an attempt to keep monitoring overhead low.\nThe metric is tagged by the name of the internal callback function being\nexecuted, as 'callback_name'. Possible values are on_partitions_assigned\nand on_partitions_revoked."}, "arroyo.consumer.run.close_strategy": {"name": "arroyo.consumer.run.close_strategy", "type": "Time", "description": "Duration metric measuring the time it took to flush in-flight messages\nand shut down the strategies."}, "arroyo.consumer.run.create_strategy": {"name": "arroyo.consumer.run.create_strategy", "type": "Time", "description": "Duration metric measuring the time it took to create the processing strategy."}, "arroyo.consumer.partitions_revoked.count": {"name": "arroyo.consumer.partitions_revoked.count", "type": "Counter", "description": "How many partitions have been revoked just now."}, "arroyo.consumer.partitions_assigned.count": {"name": "arroyo.consumer.partitions_assigned.count", "type": "Counter", "description": "How many partitions have been assigned just now."}, "arroyo.consumer.latency": {"name": "arroyo.consumer.latency", "type": "Time", "description": "Consumer latency in seconds. Recorded by the commit offsets strategy."}, "arroyo.consumer.pause": {"name": "arroyo.consumer.pause", "type": "Counter", "description": "Metric for when the underlying rdkafka consumer is being paused.\nThis flushes internal prefetch buffers."}, "arroyo.consumer.resume": {"name": "arroyo.consumer.resume", "type": "Counter", "description": "Metric for when the underlying rdkafka consumer is being resumed.\nThis might cause increased network usage as messages are being re-fetched."}, "arroyo.consumer.librdkafka.total_queue_size": {"name": "arroyo.consumer.librdkafka.total_queue_size", "type": "Gauge", "description": "Queue size of background queue that librdkafka uses to prefetch messages."}, "arroyo.processing.strategies.healthcheck.touch": {"name": "arroyo.processing.strategies.healthcheck.touch", "type": "Counter", "description": "Counter metric to measure how often the healthcheck file has been touched."}, "arroyo.strategies.filter.dropped_messages": {"name": "arroyo.strategies.filter.dropped_messages", "type": "Counter", "description": "Number of messages dropped in the FilterStep strategy"}, "arroyo.consumer.dlq.dropped_messages": {"name": "arroyo.consumer.dlq.dropped_messages", "type": "Counter", "description": "how many messages are dropped due to errors producing to the dlq"}, "arroyo.consumer.dlq_buffer.len": {"name": "arroyo.consumer.dlq_buffer.len", "type": "Gauge", "description": "Current length of the DLQ buffer deque"}, "arroyo.consumer.dlq_buffer.exceeded": {"name": "arroyo.consumer.dlq_buffer.exceeded", "type": "Counter", "description": "Number of times the DLQ buffer size has been exceeded, causing messages to be dropped"}, "arroyo.consumer.dlq_buffer.assigned_partitions": {"name": "arroyo.consumer.dlq_buffer.assigned_partitions", "type": "Gauge", "description": "Number of partitions being tracked in the DLQ buffer"}}
+{"arroyo.strategies.run_task_with_multiprocessing.batch.size.msg": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.size.msg", "type": "Time", "description": "Number of messages in a multiprocessing batch"}, "arroyo.strategies.run_task_with_multiprocessing.batch.size.bytes": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.size.bytes", "type": "Time", "description": "Number of bytes in a multiprocessing batch"}, "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.msg": {"name": "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.msg", "type": "Time", "description": "Number of messages in a multiprocessing batch after the message transformation"}, "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.bytes": {"name": "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.bytes", "type": "Time", "description": "Number of bytes in a multiprocessing batch after the message transformation"}, "arroyo.consumer.run.count": {"name": "arroyo.consumer.run.count", "type": "Counter", "description": "Number of times the consumer is spinning"}, "arroyo.consumer.invalid_message.count": {"name": "arroyo.consumer.invalid_message.count", "type": "Counter", "description": "Number of times the consumer encountered an invalid message."}, "arroyo.strategies.reduce.batch_time": {"name": "arroyo.strategies.reduce.batch_time", "type": "Time", "description": "How long it took the Reduce step to fill up a batch"}, "arroyo.strategies.run_task_with_multiprocessing.batch.backpressure": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.backpressure", "type": "Counter", "description": "Incremented when a strategy after multiprocessing applies\nbackpressure to multiprocessing. May be a reason why CPU cannot be\nsaturated."}, "arroyo.strategies.run_task_with_multiprocessing.batch.input.overflow": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.input.overflow", "type": "Counter", "description": "Incremented when multiprocessing cannot fill the input batch\nbecause not enough memory was allocated. This results in batches smaller\nthan configured. Increase `input_block_size` to fix."}, "arroyo.strategies.run_task_with_multiprocessing.batch.output.overflow": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.output.overflow", "type": "Counter", "description": "Incremented when multiprocessing cannot pull results in batches\nequal to the input batch size, because not enough memory was allocated.\nThis can be devastating for throughput. Increase `output_block_size` to\nfix."}, "arroyo.strategies.run_task_with_multiprocessing.batch.input.resize": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.input.resize", "type": "Counter", "description": "Arroyo has decided to re-allocate a block in order to combat input\nbuffer overflow. This behavior can be disabled by explicitly setting\n`input_block_size` to a not-None value in `RunTaskWithMultiprocessing`."}, "arroyo.strategies.run_task_with_multiprocessing.batch.output.resize": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.output.resize", "type": "Counter", "description": "Arroyo has decided to re-allocate a block in order to combat output\nbuffer overflow. This behavior can be disabled by explicitly setting\n`output_block_size` to a not-None value in `RunTaskWithMultiprocessing`."}, "arroyo.strategies.run_task_with_multiprocessing.batches_in_progress": {"name": "arroyo.strategies.run_task_with_multiprocessing.batches_in_progress", "type": "Gauge", "description": "How many batches are being processed in parallel by multiprocessing."}, "arroyo.strategies.run_task_with_multiprocessing.processes": {"name": "arroyo.strategies.run_task_with_multiprocessing.processes", "type": "Counter", "description": "A subprocess by multiprocessing unexpectedly died.\n\"sigchld.detected\",\nGauge: Shows how many processes the multiprocessing strategy is\nconfigured with."}, "arroyo.strategies.run_task_with_multiprocessing.pool.create": {"name": "arroyo.strategies.run_task_with_multiprocessing.pool.create", "type": "Counter", "description": "Incremented when the multiprocessing pool is created (or re-created)."}, "arroyo.consumer.poll.time": {"name": "arroyo.consumer.poll.time", "type": "Time", "description": "(unitless) spent polling librdkafka for new messages."}, "arroyo.consumer.processing.time": {"name": "arroyo.consumer.processing.time", "type": "Time", "description": "(unitless) spent in strategies (blocking in strategy.submit or\nstrategy.poll)"}, "arroyo.consumer.backpressure.time": {"name": "arroyo.consumer.backpressure.time", "type": "Time", "description": "(unitless) spent pausing the consumer due to backpressure (MessageRejected)"}, "arroyo.consumer.dlq.time": {"name": "arroyo.consumer.dlq.time", "type": "Time", "description": "(unitless) spent in handling `InvalidMessage` exceptions and sending\nmessages to the the DLQ."}, "arroyo.consumer.join.time": {"name": "arroyo.consumer.join.time", "type": "Time", "description": "(unitless) spent in waiting for the strategy to exit, such as during\nshutdown or rebalancing."}, "arroyo.consumer.callback.time": {"name": "arroyo.consumer.callback.time", "type": "Time", "description": "(unitless) spent in librdkafka callbacks. This metric's timings\noverlap other timings, and might spike at the same time."}, "arroyo.consumer.shutdown.time": {"name": "arroyo.consumer.shutdown.time", "type": "Time", "description": "(unitless) spent in shutting down the consumer. This metric's\ntimings overlap other timings, and might spike at the same time."}, "arroyo.consumer.run.callback": {"name": "arroyo.consumer.run.callback", "type": "Time", "description": "A regular duration metric where each datapoint is measuring the time it\ntook to execute a single callback. This metric is distinct from the\narroyo.consumer.*.time metrics as it does not attempt to accumulate time\nspent per second in an attempt to keep monitoring overhead low.\nThe metric is tagged by the name of the internal callback function being\nexecuted, as 'callback_name'. Possible values are on_partitions_assigned\nand on_partitions_revoked."}, "arroyo.consumer.run.close_strategy": {"name": "arroyo.consumer.run.close_strategy", "type": "Time", "description": "Duration metric measuring the time it took to flush in-flight messages\nand shut down the strategies."}, "arroyo.consumer.run.create_strategy": {"name": "arroyo.consumer.run.create_strategy", "type": "Time", "description": "Duration metric measuring the time it took to create the processing strategy."}, "arroyo.consumer.partitions_revoked.count": {"name": "arroyo.consumer.partitions_revoked.count", "type": "Counter", "description": "How many partitions have been revoked just now."}, "arroyo.consumer.partitions_assigned.count": {"name": "arroyo.consumer.partitions_assigned.count", "type": "Counter", "description": "How many partitions have been assigned just now."}, "arroyo.consumer.latency": {"name": "arroyo.consumer.latency", "type": "Time", "description": "Consumer latency in seconds. Recorded by the commit offsets strategy."}, "arroyo.consumer.pause": {"name": "arroyo.consumer.pause", "type": "Counter", "description": "Metric for when the underlying rdkafka consumer is being paused.\nThis flushes internal prefetch buffers."}, "arroyo.consumer.resume": {"name": "arroyo.consumer.resume", "type": "Counter", "description": "Metric for when the underlying rdkafka consumer is being resumed.\nThis might cause increased network usage as messages are being re-fetched."}, "arroyo.consumer.librdkafka.total_queue_size": {"name": "arroyo.consumer.librdkafka.total_queue_size", "type": "Gauge", "description": "Queue size of background queue that librdkafka uses to prefetch messages."}, "arroyo.processing.strategies.healthcheck.touch": {"name": "arroyo.processing.strategies.healthcheck.touch", "type": "Counter", "description": "Counter metric to measure how often the healthcheck file has been touched."}, "arroyo.strategies.filter.dropped_messages": {"name": "arroyo.strategies.filter.dropped_messages", "type": "Counter", "description": "Number of messages dropped in the FilterStep strategy"}, "arroyo.consumer.dlq.dropped_messages": {"name": "arroyo.consumer.dlq.dropped_messages", "type": "Counter", "description": "how many messages are dropped due to errors producing to the dlq"}, "arroyo.consumer.dlq_buffer.len": {"name": "arroyo.consumer.dlq_buffer.len", "type": "Gauge", "description": "Current length of the DLQ buffer deque"}, "arroyo.consumer.dlq_buffer.exceeded": {"name": "arroyo.consumer.dlq_buffer.exceeded", "type": "Counter", "description": "Number of times the DLQ buffer size has been exceeded, causing messages to be dropped"}, "arroyo.consumer.dlq_buffer.assigned_partitions": {"name": "arroyo.consumer.dlq_buffer.assigned_partitions", "type": "Gauge", "description": "Number of partitions being tracked in the DLQ buffer"}, "arroyo.producer.librdkafka.p99_int_latency": {"name": "arroyo.producer.librdkafka.p99_int_latency", "type": "Time", "description": "Internal producer queue latency from librdkafka statistics.\nTagged by broker_id."}, "arroyo.producer.librdkafka.p99_outbuf_latency": {"name": "arroyo.producer.librdkafka.p99_outbuf_latency", "type": "Time", "description": "Output buffer latency from librdkafka statistics.\nTagged by broker_id."}, "arroyo.producer.librdkafka.p99_rtt": {"name": "arroyo.producer.librdkafka.p99_rtt", "type": "Time", "description": "Round-trip time to brokers from librdkafka statistics.\nTagged by broker_id."}, "arroyo.producer.librdkafka.avg_int_latency": {"name": "arroyo.producer.librdkafka.avg_int_latency", "type": "Time", "description": "Average internal producer queue latency from librdkafka statistics.\nTagged by broker_id."}, "arroyo.producer.librdkafka.avg_outbuf_latency": {"name": "arroyo.producer.librdkafka.avg_outbuf_latency", "type": "Time", "description": "Average output buffer latency from librdkafka statistics.\nTagged by broker_id."}, "arroyo.producer.librdkafka.avg_rtt": {"name": "arroyo.producer.librdkafka.avg_rtt", "type": "Time", "description": "Average round-trip time to brokers from librdkafka statistics.\nTagged by broker_id."}}

{sentry_arroyo-2.24.0 → sentry_arroyo-2.26.0}/arroyo/utils/metric_defs.py

@@ -106,4 +106,22 @@ MetricName = Literal[
     "arroyo.consumer.dlq_buffer.exceeded",
     # Gauge: Number of partitions being tracked in the DLQ buffer
     "arroyo.consumer.dlq_buffer.assigned_partitions",
+    # Time: Internal producer queue latency from librdkafka statistics.
+    # Tagged by broker_id.
+    "arroyo.producer.librdkafka.p99_int_latency",
+    # Time: Output buffer latency from librdkafka statistics.
+    # Tagged by broker_id.
+    "arroyo.producer.librdkafka.p99_outbuf_latency",
+    # Time: Round-trip time to brokers from librdkafka statistics.
+    # Tagged by broker_id.
+    "arroyo.producer.librdkafka.p99_rtt",
+    # Time: Average internal producer queue latency from librdkafka statistics.
+    # Tagged by broker_id.
+    "arroyo.producer.librdkafka.avg_int_latency",
+    # Time: Average output buffer latency from librdkafka statistics.
+    # Tagged by broker_id.
+    "arroyo.producer.librdkafka.avg_outbuf_latency",
+    # Time: Average round-trip time to brokers from librdkafka statistics.
+    # Tagged by broker_id.
+    "arroyo.producer.librdkafka.avg_rtt",
 ]

{sentry_arroyo-2.24.0 → sentry_arroyo-2.26.0}/arroyo/utils/metrics.py

@@ -45,6 +45,45 @@ class Metrics(Protocol):
         raise NotImplementedError
 
 
+class ConsumerMetricsWrapper(Metrics):
+    """
+    A wrapper around a metrics backend that automatically adds consumer_member_id
+    to all metrics calls.
+
+    Right now we only use this to add tags to the metrics emitted by
+    StreamProcessor, but ideally all metrics, even those emitted by strategies
+    and application code, would get this tag. The metrics abstraction in arroyo
+    is not sufficient for this. We'd have to add a "add_global_tags" method
+    (similar to the concept of global tags in sentry) and users would have to
+    implement it.
+    """
+
+    def __init__(self, metrics: Metrics) -> None:
+        self.__metrics = metrics
+        self.consumer_member_id = ""
+
+    def _add_consumer_tag(self, tags: Optional[Tags]) -> Tags:
+        return {**(tags or {}), "consumer_member_id": self.consumer_member_id}
+
+    def increment(
+        self,
+        name: MetricName,
+        value: Union[int, float] = 1,
+        tags: Optional[Tags] = None,
+    ) -> None:
+        self.__metrics.increment(name, value, tags=self._add_consumer_tag(tags))
+
+    def gauge(
+        self, name: MetricName, value: Union[int, float], tags: Optional[Tags] = None
+    ) -> None:
+        self.__metrics.gauge(name, value, tags=self._add_consumer_tag(tags))
+
+    def timing(
+        self, name: MetricName, value: Union[int, float], tags: Optional[Tags] = None
+    ) -> None:
+        self.__metrics.timing(name, value, tags=self._add_consumer_tag(tags))
+
+
 class DummyMetricsBackend(Metrics):
     """
     Default metrics backend that does not record anything.
@@ -133,4 +172,12 @@ def get_metrics() -> Metrics:
     return _metrics_backend
 
 
-__all__ = ["configure_metrics", "Metrics", "MetricName"]
+def get_consumer_metrics() -> ConsumerMetricsWrapper:
+    """
+    Get a metrics backend that automatically adds consumer_member_id to all metrics.
+    """
+    base_metrics = get_metrics()
+    return ConsumerMetricsWrapper(base_metrics)
+
+
+__all__ = ["configure_metrics", "Metrics", "MetricName", "Tags", "get_consumer_metrics"]

{sentry_arroyo-2.24.0 → sentry_arroyo-2.26.0/sentry_arroyo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.24.0
+Version: 2.26.0
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.24.0 → sentry_arroyo-2.26.0}/sentry_arroyo.egg-info/SOURCES.txt

@@ -65,6 +65,7 @@ tests/backends/__init__.py
 tests/backends/mixins.py
 tests/backends/test_commit.py
 tests/backends/test_kafka.py
+tests/backends/test_kafka_producer.py
 tests/backends/test_local.py
 tests/processing/__init__.py
 tests/processing/test_processor.py

{sentry_arroyo-2.24.0 → sentry_arroyo-2.26.0}/setup.py

@@ -10,7 +10,7 @@ def get_requirements() -> Sequence[str]:
 
 setup(
     name="sentry-arroyo",
-    version="2.24.0",
+    version="2.26.0",
     author="Sentry",
     author_email="oss@sentry.io",
     license="Apache-2.0",

sentry_arroyo-2.26.0/tests/backends/test_kafka_producer.py

@@ -0,0 +1,125 @@
+import json
+from unittest import mock
+
+from arroyo.backends.kafka.configuration import producer_stats_callback
+
+
+@mock.patch("arroyo.backends.kafka.configuration.get_metrics")
+def test_producer_stats_callback_with_both_latencies(
+    mock_get_metrics: mock.Mock,
+) -> None:
+    mock_metrics = mock.Mock()
+    mock_get_metrics.return_value = mock_metrics
+
+    stats_json = json.dumps(
+        {
+            "brokers": {
+                "1": {
+                    "int_latency": {"p99": 2000, "avg": 1000},
+                    "outbuf_latency": {"p99": 4000, "avg": 2000},
+                }
+            }
+        }
+    )
+
+    producer_stats_callback(stats_json)
+
+    assert mock_metrics.timing.call_count == 4
+    mock_metrics.timing.assert_any_call(
+        "arroyo.producer.librdkafka.p99_int_latency",
+        2.0,
+        tags={"broker_id": "1"},
+    )
+    mock_metrics.timing.assert_any_call(
+        "arroyo.producer.librdkafka.avg_int_latency",
+        1.0,
+        tags={"broker_id": "1"},
+    )
+    mock_metrics.timing.assert_any_call(
+        "arroyo.producer.librdkafka.p99_outbuf_latency",
+        4.0,
+        tags={"broker_id": "1"},
+    )
+    mock_metrics.timing.assert_any_call(
+        "arroyo.producer.librdkafka.avg_outbuf_latency",
+        2.0,
+        tags={"broker_id": "1"},
+    )
+
+
+@mock.patch("arroyo.backends.kafka.configuration.get_metrics")
+def test_producer_stats_callback_no_brokers(mock_get_metrics: mock.Mock) -> None:
+    mock_metrics = mock.Mock()
+    mock_get_metrics.return_value = mock_metrics
+
+    stats_json = json.dumps({})
+
+    producer_stats_callback(stats_json)
+
+    mock_metrics.timing.assert_not_called()
+
+
+@mock.patch("arroyo.backends.kafka.configuration.get_metrics")
+def test_producer_stats_callback_empty_broker_stats(
+    mock_get_metrics: mock.Mock,
+) -> None:
+    mock_metrics = mock.Mock()
+    mock_get_metrics.return_value = mock_metrics
+
+    stats_json = json.dumps({"brokers": {"1": {}}})
+
+    producer_stats_callback(stats_json)
+
+    mock_metrics.timing.assert_not_called()
+
+
+@mock.patch("arroyo.backends.kafka.configuration.get_metrics")
+def test_producer_stats_callback_with_all_metrics(mock_get_metrics: mock.Mock) -> None:
+    mock_metrics = mock.Mock()
+    mock_get_metrics.return_value = mock_metrics
+
+    stats_json = json.dumps(
+        {
+            "brokers": {
+                "1": {
+                    "int_latency": {"p99": 2000, "avg": 1000},  # 2/1 milliseconds
+                    "outbuf_latency": {"p99": 4000, "avg": 2000},  # 4/2 milliseconds
+                    "rtt": {"p99": 1500, "avg": 750},  # 1.5/0.75 milliseconds
+                }
+            }
+        }
+    )
+
+    producer_stats_callback(stats_json)
+
+    assert mock_metrics.timing.call_count == 6
+    mock_metrics.timing.assert_any_call(
+        "arroyo.producer.librdkafka.p99_int_latency",
+        2.0,
+        tags={"broker_id": "1"},
+    )
+    mock_metrics.timing.assert_any_call(
+        "arroyo.producer.librdkafka.avg_int_latency",
+        1.0,
+        tags={"broker_id": "1"},
+    )
+    mock_metrics.timing.assert_any_call(
+        "arroyo.producer.librdkafka.p99_outbuf_latency",
+        4.0,
+        tags={"broker_id": "1"},
+    )
+    mock_metrics.timing.assert_any_call(
+        "arroyo.producer.librdkafka.avg_outbuf_latency",
+        2.0,
+        tags={"broker_id": "1"},
+    )
+    mock_metrics.timing.assert_any_call(
+        "arroyo.producer.librdkafka.p99_rtt",
+        1.5,
+        tags={"broker_id": "1"},
+    )
+    mock_metrics.timing.assert_any_call(
+        "arroyo.producer.librdkafka.avg_rtt",
+        0.75,
+        tags={"broker_id": "1"},
+    )

sentry_arroyo-2.26.0/tests/utils/test_metrics.py

@@ -0,0 +1,84 @@
+import pytest
+
+from arroyo.utils.metrics import (
+    Gauge,
+    MetricName,
+    configure_metrics,
+    get_consumer_metrics,
+    get_metrics,
+)
+from tests.metrics import Gauge as GaugeCall
+from tests.metrics import (
+    Increment,
+    TestingMetricsBackend,
+    Timing,
+    _TestingMetricsBackend,
+)
+
+
+def test_gauge_simple() -> None:
+    backend = TestingMetricsBackend
+
+    name: MetricName = "name"  # type: ignore
+    tags = {"tag": "value"}
+    gauge = Gauge(backend, name, tags)
+
+    with gauge:
+        pass
+
+    assert backend.calls == [
+        GaugeCall(name, 0.0, tags),
+        GaugeCall(name, 1.0, tags),
+        GaugeCall(name, 0.0, tags),
+    ]
+
+
+def test_configure_metrics() -> None:
+    assert get_metrics() == TestingMetricsBackend
+
+    with pytest.raises(AssertionError):
+        configure_metrics(_TestingMetricsBackend())
+
+    # Can be reset to something else with force
+    configure_metrics(_TestingMetricsBackend(), force=True)
+    assert get_metrics() != TestingMetricsBackend
+
+
+def test_consumer_metrics_wrapper() -> None:
+    """Test that ConsumerMetricsWrapper automatically adds consumer_member_id to all metrics."""
+    # Reset to a fresh backend
+    backend = _TestingMetricsBackend()
+    configure_metrics(backend, force=True)
+
+    consumer_member_id = "test-consumer-123"
+    consumer_metrics = get_consumer_metrics()
+    consumer_metrics.consumer_member_id = consumer_member_id
+
+    # Test increment
+    consumer_metrics.increment("arroyo.consumer.run.count", 5, tags={"extra": "tag"})
+
+    # Test gauge
+    consumer_metrics.gauge("arroyo.consumer.librdkafka.total_queue_size", 10.5)
+
+    # Test timing
+    consumer_metrics.timing("arroyo.consumer.poll.time", 100, tags={"another": "tag"})
+
+    expected_calls = [
+        Increment(
+            "arroyo.consumer.run.count",
+            5,
+            {"consumer_member_id": consumer_member_id, "extra": "tag"},
+        ),
+        GaugeCall(
+            "arroyo.consumer.librdkafka.total_queue_size",
+            10.5,
+            {"consumer_member_id": consumer_member_id},
+        ),
+        Timing(
+            "arroyo.consumer.poll.time",
+            100,
+            {"consumer_member_id": consumer_member_id, "another": "tag"},
+        ),
+    ]
+
+    assert backend.calls == expected_calls

sentry_arroyo-2.24.0/tests/utils/test_metrics.py

@@ -1,33 +0,0 @@
-import pytest
-
-from arroyo.utils.metrics import Gauge, MetricName, configure_metrics, get_metrics
-from tests.metrics import Gauge as GaugeCall
-from tests.metrics import TestingMetricsBackend, _TestingMetricsBackend
-
-
-def test_gauge_simple() -> None:
-    backend = TestingMetricsBackend
-
-    name: MetricName = "name"  # type: ignore
-    tags = {"tag": "value"}
-    gauge = Gauge(backend, name, tags)
-
-    with gauge:
-        pass
-
-    assert backend.calls == [
-        GaugeCall(name, 0.0, tags),
-        GaugeCall(name, 1.0, tags),
-        GaugeCall(name, 0.0, tags),
-    ]
-
-
-def test_configure_metrics() -> None:
-    assert get_metrics() == TestingMetricsBackend
-
-    with pytest.raises(AssertionError):
-        configure_metrics(_TestingMetricsBackend())
-
-    # Can be reset to something else with force
-    configure_metrics(_TestingMetricsBackend(), force=True)
-    assert get_metrics() != TestingMetricsBackend