sentry-arroyo 2.19.11__tar.gz → 2.20.0__tar.gz

{sentry_arroyo-2.19.11/sentry_arroyo.egg-info → sentry_arroyo-2.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sentry-arroyo
-Version: 2.19.11
+Version: 2.20.0
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.19.11 → sentry_arroyo-2.20.0}/arroyo/dlq.py

@@ -28,6 +28,7 @@ from arroyo.types import (
     TStrategyPayload,
     Value,
 )
+from arroyo.utils.metrics import get_metrics
 
 logger = logging.getLogger(__name__)
 
@@ -287,6 +288,11 @@ class BufferedMessages(Generic[TStrategyPayload]):
         self.__buffered_messages: MutableMapping[
             Partition, Deque[BrokerValue[TStrategyPayload]]
         ] = defaultdict(deque)
+        self.__metrics = get_metrics()
+
+    def report_partition_metrics(self, buffered: Deque[BrokerValue[TStrategyPayload]]) -> None:
+
+        self.__metrics.gauge("arroyo.consumer.dlq_buffer.len", len(buffered))
 
     def append(self, message: BrokerValue[TStrategyPayload]) -> None:
         """
@@ -304,6 +310,7 @@ class BufferedMessages(Generic[TStrategyPayload]):
                 buffered.popleft()
 
         self.__buffered_messages[message.partition].append(message)
+        self.report_partition_metrics(self.__buffered_messages[message.partition])
 
     def pop(
         self, partition: Partition, offset: int
@@ -317,9 +324,14 @@ class BufferedMessages(Generic[TStrategyPayload]):
 
             while buffered:
                 if buffered[0].offset == offset:
-                    return buffered.popleft()
+                    msg = buffered.popleft()
+                    self.report_partition_metrics(buffered)
+                    return msg
                 if buffered[0].offset > offset:
+                    self.report_partition_metrics(buffered)
                     break
+
+                self.report_partition_metrics(buffered)
                 self.__buffered_messages[partition].popleft()
 
             return None

{sentry_arroyo-2.19.11 → sentry_arroyo-2.20.0}/arroyo/processing/processor.py

@@ -84,6 +84,7 @@ ConsumerCounter = Literal[
     "arroyo.consumer.invalid_message.count",
     "arroyo.consumer.pause",
     "arroyo.consumer.resume",
+    "arroyo.consumer.dlq.dropped_messages",
 ]
 
 
@@ -377,7 +378,11 @@ class StreamProcessor(Generic[TStrategyPayload]):
                 ) from None
 
             # XXX: This blocks if there are more than MAX_PENDING_FUTURES in the queue.
-            self.__dlq_policy.produce(invalid_message, exc.reason)
+            try:
+                self.__dlq_policy.produce(invalid_message, exc.reason)
+            except Exception:
+                logger.exception(f"Failed to produce message (partition: {exc.partition} offset: {exc.offset}) to DLQ topic, dropping")
+                self.__metrics_buffer.incr_counter("arroyo.consumer.dlq.dropped_messages", 1)
 
             self.__metrics_buffer.incr_timing(
                 "arroyo.consumer.dlq.time", time.time() - start_dlq

{sentry_arroyo-2.19.11 → sentry_arroyo-2.20.0}/arroyo/processing/strategies/unfold.py

@@ -3,7 +3,7 @@ from collections import deque
 from typing import Callable, Deque, Generic, Iterable, Optional, TypeVar, Union, cast
 
 from arroyo.processing.strategies.abstract import MessageRejected, ProcessingStrategy
-from arroyo.types import BaseValue, FilteredPayload, Message
+from arroyo.types import BaseValue, FilteredPayload, Message, Value
 
 TInput = TypeVar("TInput")
 TOutput = TypeVar("TOutput")
@@ -17,9 +17,20 @@ class Unfold(
     messages submitting them one by one to the next step. The generated
     messages are created according to the generator function provided by the user.
 
+    ::
+
+        def generator(num: int) -> Sequence[Value[int]]:
+            return [Value(i, {}, None) for i in range(num)]
+
+        unfold = Unfold(generator, next_step)
+
     The generator function provided must return an iterable (i.e. a class that
     implements `__iter__` ).
 
+    The generator can choose to set its own committable on the return value. If
+    the `committable` is empty, `Unfold` will use the offsets of the
+    original message.
+
     If this step receives a `MessageRejected` exception from the next
     step it keeps the remaining messages and attempts to submit
     them on subsequent calls to `poll`
@@ -47,12 +58,23 @@ class Unfold(
             )
             return
 
-        iterable = self.__generator(message.payload)
+        iterable = list(self.__generator(message.payload))
+        num_messages = len(iterable)
 
         store_remaining_messages = False
 
-        for value in iterable:
+        for i, value in enumerate(iterable):
             next_message = Message(value=value)
+            # If generator did not provide committable, patch our own
+            # committable onto it
+            if i == num_messages - 1 and not next_message.committable:
+                next_message = Message(
+                    Value(
+                        next_message.payload,
+                        message.committable,
+                        next_message.timestamp,
+                    )
+                )
 
             if store_remaining_messages == False:
                 try:

{sentry_arroyo-2.19.11 → sentry_arroyo-2.20.0}/arroyo/utils/metricDefs.json

@@ -1 +1 @@
-{"arroyo.strategies.run_task_with_multiprocessing.batch.size.msg": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.size.msg", "type": "Time", "description": "Number of messages in a multiprocessing batch"}, "arroyo.strategies.run_task_with_multiprocessing.batch.size.bytes": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.size.bytes", "type": "Time", "description": "Number of bytes in a multiprocessing batch"}, "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.msg": {"name": "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.msg", "type": "Time", "description": "Number of messages in a multiprocessing batch after the message transformation"}, "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.bytes": {"name": "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.bytes", "type": "Time", "description": "Number of bytes in a multiprocessing batch after the message transformation"}, "arroyo.consumer.run.count": {"name": "arroyo.consumer.run.count", "type": "Counter", "description": "Number of times the consumer is spinning"}, "arroyo.consumer.invalid_message.count": {"name": "arroyo.consumer.invalid_message.count", "type": "Counter", "description": "Number of times the consumer encountered an invalid message."}, "arroyo.strategies.reduce.batch_time": {"name": "arroyo.strategies.reduce.batch_time", "type": "Time", "description": "How long it took the Reduce step to fill up a batch"}, "arroyo.strategies.run_task_with_multiprocessing.batch.backpressure": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.backpressure", "type": "Counter", "description": "Incremented when a strategy after multiprocessing applies\nbackpressure to multiprocessing. May be a reason why CPU cannot be\nsaturated."}, "arroyo.strategies.run_task_with_multiprocessing.batch.input.overflow": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.input.overflow", "type": "Counter", "description": "Incremented when multiprocessing cannot fill the input batch\nbecause not enough memory was allocated. This results in batches smaller\nthan configured. Increase `input_block_size` to fix."}, "arroyo.strategies.run_task_with_multiprocessing.batch.output.overflow": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.output.overflow", "type": "Counter", "description": "Incremented when multiprocessing cannot pull results in batches\nequal to the input batch size, because not enough memory was allocated.\nThis can be devastating for throughput. Increase `output_block_size` to\nfix."}, "arroyo.strategies.run_task_with_multiprocessing.batch.input.resize": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.input.resize", "type": "Counter", "description": "Arroyo has decided to re-allocate a block in order to combat input\nbuffer overflow. This behavior can be disabled by explicitly setting\n`input_block_size` to a not-None value in `RunTaskWithMultiprocessing`."}, "arroyo.strategies.run_task_with_multiprocessing.batch.output.resize": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.output.resize", "type": "Counter", "description": "Arroyo has decided to re-allocate a block in order to combat output\nbuffer overflow. This behavior can be disabled by explicitly setting\n`output_block_size` to a not-None value in `RunTaskWithMultiprocessing`."}, "arroyo.strategies.run_task_with_multiprocessing.batches_in_progress": {"name": "arroyo.strategies.run_task_with_multiprocessing.batches_in_progress", "type": "Gauge", "description": "How many batches are being processed in parallel by multiprocessing."}, "arroyo.strategies.run_task_with_multiprocessing.processes": {"name": "arroyo.strategies.run_task_with_multiprocessing.processes", "type": "Counter", "description": "A subprocess by multiprocessing unexpectedly died.\n\"sigchld.detected\",\nGauge: Shows how many processes the multiprocessing strategy is\nconfigured with."}, "arroyo.strategies.run_task_with_multiprocessing.pool.create": {"name": "arroyo.strategies.run_task_with_multiprocessing.pool.create", "type": "Counter", "description": "Incremented when the multiprocessing pool is created (or re-created)."}, "arroyo.consumer.poll.time": {"name": "arroyo.consumer.poll.time", "type": "Time", "description": "(unitless) spent polling librdkafka for new messages."}, "arroyo.consumer.processing.time": {"name": "arroyo.consumer.processing.time", "type": "Time", "description": "(unitless) spent in strategies (blocking in strategy.submit or\nstrategy.poll)"}, "arroyo.consumer.backpressure.time": {"name": "arroyo.consumer.backpressure.time", "type": "Time", "description": "(unitless) spent pausing the consumer due to backpressure (MessageRejected)"}, "arroyo.consumer.dlq.time": {"name": "arroyo.consumer.dlq.time", "type": "Time", "description": "(unitless) spent in handling `InvalidMessage` exceptions and sending\nmessages to the the DLQ."}, "arroyo.consumer.join.time": {"name": "arroyo.consumer.join.time", "type": "Time", "description": "(unitless) spent in waiting for the strategy to exit, such as during\nshutdown or rebalancing."}, "arroyo.consumer.callback.time": {"name": "arroyo.consumer.callback.time", "type": "Time", "description": "(unitless) spent in librdkafka callbacks. This metric's timings\noverlap other timings, and might spike at the same time."}, "arroyo.consumer.shutdown.time": {"name": "arroyo.consumer.shutdown.time", "type": "Time", "description": "(unitless) spent in shutting down the consumer. This metric's\ntimings overlap other timings, and might spike at the same time."}, "arroyo.consumer.run.callback": {"name": "arroyo.consumer.run.callback", "type": "Time", "description": "A regular duration metric where each datapoint is measuring the time it\ntook to execute a single callback. This metric is distinct from the\narroyo.consumer.*.time metrics as it does not attempt to accumulate time\nspent per second in an attempt to keep monitoring overhead low.\nThe metric is tagged by the name of the internal callback function being\nexecuted, as 'callback_name'. Possible values are on_partitions_assigned\nand on_partitions_revoked."}, "arroyo.consumer.run.close_strategy": {"name": "arroyo.consumer.run.close_strategy", "type": "Time", "description": "Duration metric measuring the time it took to flush in-flight messages\nand shut down the strategies."}, "arroyo.consumer.run.create_strategy": {"name": "arroyo.consumer.run.create_strategy", "type": "Time", "description": "Duration metric measuring the time it took to create the processing strategy."}, "arroyo.consumer.partitions_revoked.count": {"name": "arroyo.consumer.partitions_revoked.count", "type": "Counter", "description": "How many partitions have been revoked just now."}, "arroyo.consumer.partitions_assigned.count": {"name": "arroyo.consumer.partitions_assigned.count", "type": "Counter", "description": "How many partitions have been assigned just now."}, "arroyo.consumer.latency": {"name": "arroyo.consumer.latency", "type": "Time", "description": "Consumer latency in seconds. Recorded by the commit offsets strategy."}, "arroyo.consumer.pause": {"name": "arroyo.consumer.pause", "type": "Counter", "description": "Metric for when the underlying rdkafka consumer is being paused.\nThis flushes internal prefetch buffers."}, "arroyo.consumer.resume": {"name": "arroyo.consumer.resume", "type": "Counter", "description": "Metric for when the underlying rdkafka consumer is being resumed.\nThis might cause increased network usage as messages are being re-fetched."}, "arroyo.consumer.librdkafka.total_queue_size": {"name": "arroyo.consumer.librdkafka.total_queue_size", "type": "Gauge", "description": "Queue size of background queue that librdkafka uses to prefetch messages."}, "arroyo.processing.strategies.healthcheck.touch": {"name": "arroyo.processing.strategies.healthcheck.touch", "type": "Counter", "description": "Counter metric to measure how often the healthcheck file has been touched."}, "arroyo.strategies.filter.dropped_messages": {"name": "arroyo.strategies.filter.dropped_messages", "type": "Counter", "description": "Number of messages dropped in the FilterStep strategy"}}
+{"arroyo.strategies.run_task_with_multiprocessing.batch.size.msg": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.size.msg", "type": "Time", "description": "Number of messages in a multiprocessing batch"}, "arroyo.strategies.run_task_with_multiprocessing.batch.size.bytes": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.size.bytes", "type": "Time", "description": "Number of bytes in a multiprocessing batch"}, "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.msg": {"name": "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.msg", "type": "Time", "description": "Number of messages in a multiprocessing batch after the message transformation"}, "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.bytes": {"name": "arroyo.strategies.run_task_with_multiprocessing.output_batch.size.bytes", "type": "Time", "description": "Number of bytes in a multiprocessing batch after the message transformation"}, "arroyo.consumer.run.count": {"name": "arroyo.consumer.run.count", "type": "Counter", "description": "Number of times the consumer is spinning"}, "arroyo.consumer.invalid_message.count": {"name": "arroyo.consumer.invalid_message.count", "type": "Counter", "description": "Number of times the consumer encountered an invalid message."}, "arroyo.strategies.reduce.batch_time": {"name": "arroyo.strategies.reduce.batch_time", "type": "Time", "description": "How long it took the Reduce step to fill up a batch"}, "arroyo.strategies.run_task_with_multiprocessing.batch.backpressure": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.backpressure", "type": "Counter", "description": "Incremented when a strategy after multiprocessing applies\nbackpressure to multiprocessing. May be a reason why CPU cannot be\nsaturated."}, "arroyo.strategies.run_task_with_multiprocessing.batch.input.overflow": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.input.overflow", "type": "Counter", "description": "Incremented when multiprocessing cannot fill the input batch\nbecause not enough memory was allocated. This results in batches smaller\nthan configured. Increase `input_block_size` to fix."}, "arroyo.strategies.run_task_with_multiprocessing.batch.output.overflow": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.output.overflow", "type": "Counter", "description": "Incremented when multiprocessing cannot pull results in batches\nequal to the input batch size, because not enough memory was allocated.\nThis can be devastating for throughput. Increase `output_block_size` to\nfix."}, "arroyo.strategies.run_task_with_multiprocessing.batch.input.resize": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.input.resize", "type": "Counter", "description": "Arroyo has decided to re-allocate a block in order to combat input\nbuffer overflow. This behavior can be disabled by explicitly setting\n`input_block_size` to a not-None value in `RunTaskWithMultiprocessing`."}, "arroyo.strategies.run_task_with_multiprocessing.batch.output.resize": {"name": "arroyo.strategies.run_task_with_multiprocessing.batch.output.resize", "type": "Counter", "description": "Arroyo has decided to re-allocate a block in order to combat output\nbuffer overflow. This behavior can be disabled by explicitly setting\n`output_block_size` to a not-None value in `RunTaskWithMultiprocessing`."}, "arroyo.strategies.run_task_with_multiprocessing.batches_in_progress": {"name": "arroyo.strategies.run_task_with_multiprocessing.batches_in_progress", "type": "Gauge", "description": "How many batches are being processed in parallel by multiprocessing."}, "arroyo.strategies.run_task_with_multiprocessing.processes": {"name": "arroyo.strategies.run_task_with_multiprocessing.processes", "type": "Counter", "description": "A subprocess by multiprocessing unexpectedly died.\n\"sigchld.detected\",\nGauge: Shows how many processes the multiprocessing strategy is\nconfigured with."}, "arroyo.strategies.run_task_with_multiprocessing.pool.create": {"name": "arroyo.strategies.run_task_with_multiprocessing.pool.create", "type": "Counter", "description": "Incremented when the multiprocessing pool is created (or re-created)."}, "arroyo.consumer.poll.time": {"name": "arroyo.consumer.poll.time", "type": "Time", "description": "(unitless) spent polling librdkafka for new messages."}, "arroyo.consumer.processing.time": {"name": "arroyo.consumer.processing.time", "type": "Time", "description": "(unitless) spent in strategies (blocking in strategy.submit or\nstrategy.poll)"}, "arroyo.consumer.backpressure.time": {"name": "arroyo.consumer.backpressure.time", "type": "Time", "description": "(unitless) spent pausing the consumer due to backpressure (MessageRejected)"}, "arroyo.consumer.dlq.time": {"name": "arroyo.consumer.dlq.time", "type": "Time", "description": "(unitless) spent in handling `InvalidMessage` exceptions and sending\nmessages to the the DLQ."}, "arroyo.consumer.join.time": {"name": "arroyo.consumer.join.time", "type": "Time", "description": "(unitless) spent in waiting for the strategy to exit, such as during\nshutdown or rebalancing."}, "arroyo.consumer.callback.time": {"name": "arroyo.consumer.callback.time", "type": "Time", "description": "(unitless) spent in librdkafka callbacks. This metric's timings\noverlap other timings, and might spike at the same time."}, "arroyo.consumer.shutdown.time": {"name": "arroyo.consumer.shutdown.time", "type": "Time", "description": "(unitless) spent in shutting down the consumer. This metric's\ntimings overlap other timings, and might spike at the same time."}, "arroyo.consumer.run.callback": {"name": "arroyo.consumer.run.callback", "type": "Time", "description": "A regular duration metric where each datapoint is measuring the time it\ntook to execute a single callback. This metric is distinct from the\narroyo.consumer.*.time metrics as it does not attempt to accumulate time\nspent per second in an attempt to keep monitoring overhead low.\nThe metric is tagged by the name of the internal callback function being\nexecuted, as 'callback_name'. Possible values are on_partitions_assigned\nand on_partitions_revoked."}, "arroyo.consumer.run.close_strategy": {"name": "arroyo.consumer.run.close_strategy", "type": "Time", "description": "Duration metric measuring the time it took to flush in-flight messages\nand shut down the strategies."}, "arroyo.consumer.run.create_strategy": {"name": "arroyo.consumer.run.create_strategy", "type": "Time", "description": "Duration metric measuring the time it took to create the processing strategy."}, "arroyo.consumer.partitions_revoked.count": {"name": "arroyo.consumer.partitions_revoked.count", "type": "Counter", "description": "How many partitions have been revoked just now."}, "arroyo.consumer.partitions_assigned.count": {"name": "arroyo.consumer.partitions_assigned.count", "type": "Counter", "description": "How many partitions have been assigned just now."}, "arroyo.consumer.latency": {"name": "arroyo.consumer.latency", "type": "Time", "description": "Consumer latency in seconds. Recorded by the commit offsets strategy."}, "arroyo.consumer.pause": {"name": "arroyo.consumer.pause", "type": "Counter", "description": "Metric for when the underlying rdkafka consumer is being paused.\nThis flushes internal prefetch buffers."}, "arroyo.consumer.resume": {"name": "arroyo.consumer.resume", "type": "Counter", "description": "Metric for when the underlying rdkafka consumer is being resumed.\nThis might cause increased network usage as messages are being re-fetched."}, "arroyo.consumer.librdkafka.total_queue_size": {"name": "arroyo.consumer.librdkafka.total_queue_size", "type": "Gauge", "description": "Queue size of background queue that librdkafka uses to prefetch messages."}, "arroyo.processing.strategies.healthcheck.touch": {"name": "arroyo.processing.strategies.healthcheck.touch", "type": "Counter", "description": "Counter metric to measure how often the healthcheck file has been touched."}, "arroyo.strategies.filter.dropped_messages": {"name": "arroyo.strategies.filter.dropped_messages", "type": "Counter", "description": "Number of messages dropped in the FilterStep strategy"}, "arroyo.consumer.dlq.dropped_messages": {"name": "arroyo.consumer.dlq.dropped_messages", "type": "Counter", "description": "how many messages are dropped due to errors producing to the dlq"}, "arroyo.consumer.dlq_buffer.len": {"name": "arroyo.consumer.dlq_buffer.len", "type": "Gauge", "description": "Current length of the DLQ buffer deque"}}

{sentry_arroyo-2.19.11 → sentry_arroyo-2.20.0}/arroyo/utils/metric_defs.py

@@ -98,4 +98,8 @@ MetricName = Literal[
     "arroyo.processing.strategies.healthcheck.touch",
     # Counter: Number of messages dropped in the FilterStep strategy
     "arroyo.strategies.filter.dropped_messages",
+    # Counter: how many messages are dropped due to errors producing to the dlq
+    "arroyo.consumer.dlq.dropped_messages",
+    # Gauge: Current length of the DLQ buffer deque
+    "arroyo.consumer.dlq_buffer.len",
 ]

{sentry_arroyo-2.19.11 → sentry_arroyo-2.20.0/sentry_arroyo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sentry-arroyo
-Version: 2.19.11
+Version: 2.20.0
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.19.11 → sentry_arroyo-2.20.0}/setup.py

@@ -10,7 +10,7 @@ def get_requirements() -> Sequence[str]:
 
 setup(
     name="sentry-arroyo",
-    version="2.19.11",
+    version="2.20.0",
     author="Sentry",
     author_email="oss@sentry.io",
     license="Apache-2.0",

{sentry_arroyo-2.19.11 → sentry_arroyo-2.20.0}/tests/processing/strategies/test_unfold.py

@@ -11,7 +11,7 @@ NOW = datetime.now()
 
 
 def generator(num: int) -> Sequence[Value[int]]:
-    return [Value(i, {PARTITION: i}, NOW) for i in range(num)]
+    return [Value(i, {}, NOW) for i in range(num)]
 
 
 def test_unfold() -> None:
@@ -23,7 +23,9 @@ def test_unfold() -> None:
     strategy.submit(message)
 
     assert next_step.submit.call_args_list == [
-        call(Message(Value(0, {PARTITION: 0}, NOW))),
+        # first message has no committable since the original message has not fully been processed
+        call(Message(Value(0, {}, NOW))),
+        # second message is last message from batch, so we can say the original msg was fully processed
         call(Message(Value(1, {PARTITION: 1}, NOW))),
     ]
 
@@ -44,7 +46,7 @@ def test_message_rejected() -> None:
 
     # Message doesn't actually go through since it was rejected
     assert next_step.submit.call_args_list == [
-        call(Message(Value(0, {PARTITION: 0}, NOW))),
+        call(Message(Value(0, {}, NOW))),
     ]
 
     # clear the side effect, both messages should be submitted now
@@ -53,7 +55,7 @@ def test_message_rejected() -> None:
     strategy.poll()
 
     assert next_step.submit.call_args_list == [
-        call(Message(Value(0, {PARTITION: 0}, NOW))),
+        call(Message(Value(0, {}, NOW))),
         call(Message(Value(1, {PARTITION: 1}, NOW))),
     ]
 

{sentry_arroyo-2.19.11 → sentry_arroyo-2.20.0}/tests/processing/test_processor.py

@@ -583,6 +583,7 @@ def test_dlq() -> None:
     assert dlq_policy.producer.produce.call_count == 1
 
 
+
 def test_healthcheck(tmpdir: py.path.local) -> None:
     """
     Test healthcheck strategy e2e with StreamProcessor, to ensure the