sentry-arroyo 2.32.3__py3-none-any.whl → 2.32.5__py3-none-any.whl

arroyo/backends/kafka/configuration.py

@@ -238,6 +238,7 @@ def build_kafka_consumer_configuration(
     override_params: Optional[Mapping[str, Any]] = None,
     strict_offset_reset: Optional[bool] = None,
     enable_auto_commit: bool = False,
+    retry_handle_destroyed: bool = False,
 ) -> KafkaBrokerConfig:
 
     if auto_offset_reset is None:
@@ -263,6 +264,8 @@ def build_kafka_consumer_configuration(
         "arroyo.strict.offset.reset": strict_offset_reset,
         # this is an arroyo specific flag to enable auto-commit mode
         "arroyo.enable.auto.commit": enable_auto_commit,
+        # arroyo specific flag to enable retries when hitting `KafkaError._DESTROY` while committing
+        "arroyo.retry.broker.handle.destroyed": retry_handle_destroyed,
         # overridden to reduce memory usage when there's a large backlog
         "queued.max.messages.kbytes": queued_max_messages_kbytes,
         "queued.min.messages": queued_min_messages,

arroyo/processing/processor.py

@@ -141,14 +141,15 @@ class StreamProcessor(Generic[TStrategyPayload]):
         dlq_policy: Optional[DlqPolicy[TStrategyPayload]] = None,
         join_timeout: Optional[float] = None,
         shutdown_strategy_before_consumer: bool = False,
+        handle_poll_while_paused: Optional[bool] = False,
     ) -> None:
         self.__consumer = consumer
         self.__processor_factory = processor_factory
         self.__metrics_buffer = MetricsBuffer()
 
-        self.__processing_strategy: Optional[ProcessingStrategy[TStrategyPayload]] = (
-            None
-        )
+        self.__processing_strategy: Optional[
+            ProcessingStrategy[TStrategyPayload]
+        ] = None
 
         self.__message: Optional[BrokerValue[TStrategyPayload]] = None
 
@@ -167,6 +168,8 @@ class StreamProcessor(Generic[TStrategyPayload]):
         self.__shutdown_requested = False
         self.__shutdown_strategy_before_consumer = shutdown_strategy_before_consumer
 
+        self.__handle_poll_while_paused = handle_poll_while_paused
+
         # Buffers messages for DLQ. Messages are added when they are submitted for processing and
         # removed once the commit callback is fired as they are guaranteed to be valid at that point.
         self.__buffered_messages: BufferedMessages[TStrategyPayload] = BufferedMessages(
@@ -464,40 +467,59 @@ class StreamProcessor(Generic[TStrategyPayload]):
                         self.__is_paused = True
 
                     elif self.__is_paused:
-                        paused_partitions = set(self.__consumer.paused())
-                        all_partitions = set(self.__consumer.tell())
-                        unpaused_partitions = (
-                            all_partitions - paused_partitions
-                        )
-                        if unpaused_partitions:
-                            logger.warning(
-                                "Processor in paused state while consumer is partially unpaused: %s, paused: %s",
-                                unpaused_partitions,
-                                paused_partitions,
-                            )
-                            self.__is_paused = False
-                            # unpause paused partitions... just in case a subset is paused
-                            self.__metrics_buffer.incr_counter(
-                                "arroyo.consumer.resume", 1
-                            )
-                            self.__consumer.resume([*paused_partitions])
-                        else:
-                            # A paused consumer should still poll periodically to avoid it's partitions
-                            # getting revoked by the broker after reaching the max.poll.interval.ms
-                            # Polling a paused consumer should never yield a message.
-                            logger.warning("consumer.tell() value right before poll() is: %s", self.__consumer.tell())
+                        if self.__handle_poll_while_paused:
                             maybe_message = self.__consumer.poll(0.1)
                             if maybe_message is not None:
-                                logger.warning("Received a message from partition: %s, \
-                                                consumer.tell() value right after poll() is: %s \
-                                                Some lines above consumer.tell() was called, all_partitons value was: %s \
-                                                Some lines above consumer.paused() was called, paused_partitions value is: %s",
-                                                maybe_message.partition,
-                                                self.__consumer.tell(),
-                                                all_partitions,
-                                                paused_partitions
-                                                )
-                            assert maybe_message is None
+                                # The paused consumer, in the above poll, has
+                                # gone through rebalancing. In this case we
+                                # expect that partition revocation cleared the
+                                # pause flag and the carried over message.
+                                # this assumption will not hold for cooperative-sticky rebalancing.
+                                assert (
+                                    not self.__is_paused
+                                ), "consumer unpaused itself without rebalancing"
+                                assert (
+                                    self.__message is None
+                                ), "consumer unpaused itself without rebalancing"
+                        # this path might raise AssertionErrors
+                        else:
+                            paused_partitions = set(self.__consumer.paused())
+                            all_partitions = set(self.__consumer.tell())
+                            unpaused_partitions = all_partitions - paused_partitions
+                            if unpaused_partitions:
+                                logger.warning(
+                                    "Processor in paused state while consumer is partially unpaused: %s, paused: %s",
+                                    unpaused_partitions,
+                                    paused_partitions,
+                                )
+                                self.__is_paused = False
+                                # unpause paused partitions... just in case a subset is paused
+                                self.__metrics_buffer.incr_counter(
+                                    "arroyo.consumer.resume", 1
+                                )
+                                self.__consumer.resume([*paused_partitions])
+                            else:
+                                # A paused consumer should still poll periodically to avoid it's partitions
+                                # getting revoked by the broker after reaching the max.poll.interval.ms
+                                # Polling a paused consumer should never yield a message.
+                                logger.warning(
+                                    "consumer.tell() value right before poll() is: %s",
+                                    self.__consumer.tell(),
+                                )
+                                maybe_message = self.__consumer.poll(0.1)
+                                if maybe_message is not None:
+                                    logger.warning(
+                                        "Received a message from partition: %s, \
+                                                    consumer.tell() value right after poll() is: %s \
+                                                    Some lines above consumer.tell() was called, all_partitons value was: %s \
+                                                    Some lines above consumer.paused() was called, paused_partitions value is: %s",
+                                        maybe_message.partition,
+                                        self.__consumer.tell(),
+                                        all_partitions,
+                                        paused_partitions,
+                                    )
+                                assert maybe_message is None
+
                     else:
                         time.sleep(0.01)
 

{sentry_arroyo-2.32.3.dist-info → sentry_arroyo-2.32.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.32.3
+Version: 2.32.5
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.32.3.dist-info → sentry_arroyo-2.32.5.dist-info}/RECORD

@@ -8,7 +8,7 @@ arroyo/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/backends/abstract.py,sha256=Wy9xhE1dtFiumG8Cz3JhksJ0rF74uJWZWq10UO1rxOI,9524
 arroyo/backends/kafka/__init__.py,sha256=xgf-AqHbQkJsh73YokO2uoyyHfZf8XwUp6BULtM8stI,445
 arroyo/backends/kafka/commit.py,sha256=LPsjvX5PPXR62DT6sa5GuSF78qk9F_L--Fz4kw7-m-s,3060
-arroyo/backends/kafka/configuration.py,sha256=zB54w7qsyVeMVkH5MpV6F8ztXfEzIXrex6aKYX-GcqA,9141
+arroyo/backends/kafka/configuration.py,sha256=42FQyrpIQGRaECBPljSaNO8E1RRyEoG1-a5cg3PwRe0,9356
 arroyo/backends/kafka/consumer.py,sha256=dzypkibGGieLg819TOoxtbVpVaxKYF9dazorQv5FxPg,34688
 arroyo/backends/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/backends/local/backend.py,sha256=hUXdCV6B5e7s4mjFC6HnIuUhjENU2tNZt5vuEOJmGZQ,13888
@@ -16,7 +16,7 @@ arroyo/backends/local/storages/__init__.py,sha256=AGYujdAAcn3osoj9jq84IzTywYbkID
 arroyo/backends/local/storages/abstract.py,sha256=1qVQp6roxHkK6XT2aklZyZk1qq7RzcPN6Db_CA5--kg,2901
 arroyo/backends/local/storages/memory.py,sha256=AoKDsVZzBXkOJyWArKWp3vfGfU9xLlKFXE9gsJiMIzQ,2613
 arroyo/processing/__init__.py,sha256=vZVg0wJvJfoVzlzGvnL59bT6YNIRJNQ5t7oU045Qbk4,87
-arroyo/processing/processor.py,sha256=raPgmhl9K9kwI9ZlhCZELJGaba3xK525gmTi4bABenc,22377
+arroyo/processing/processor.py,sha256=CHBvpAxH6T3MrUtoCJR7nTFJppqmIAx0DTosADrUAEM,23711
 arroyo/processing/strategies/__init__.py,sha256=EU_JMb54eOxMxaC5mIFpI-sAF-X2ZScbE8czBZ7bQkY,1106
 arroyo/processing/strategies/abstract.py,sha256=nu7juEz_aQmQIH35Z8u--FBuLjkK8_LQ1hIG2xpw9AA,4808
 arroyo/processing/strategies/batching.py,sha256=s89xC6lQpBseEaApu1iNTipXGKeO95OMwinj2VBKn9s,4778
@@ -46,17 +46,17 @@ examples/transform_and_produce/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 examples/transform_and_produce/batched.py,sha256=st2R6qTneAtV0JFbKP30Ti3sJDYj8Jkbmta9JckKdZU,2636
 examples/transform_and_produce/script.py,sha256=8kSMIjQNqGYEVyE0PvrfJh-a_UYCrJSstTp_De7kyyg,2306
 examples/transform_and_produce/simple.py,sha256=H7xqxItjl4tx34wVW5dy6mB9G39QucAtxkJSBzVmjgA,1637
-sentry_arroyo-2.32.3.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
+sentry_arroyo-2.32.5.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
 tests/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/backends/mixins.py,sha256=sfNyE0VTeiD3GHOnBYl-9urvPuURI2G1BWke0cz7Dvc,20445
 tests/backends/test_commit.py,sha256=iTHfK1qsBxim0XwxgMvNNSMqDUMEHoYkYBDcgxGBFbs,831
 tests/backends/test_confluent_producer.py,sha256=KWqgvjDvqAdd0HxngdWKsUJaV7Hl1L5vAVQhBYlHeHU,3146
-tests/backends/test_kafka.py,sha256=wBFCKEHoP6h0uG1bgDuzk84IZmrV_UVOFCrtbxztmJg,15506
+tests/backends/test_kafka.py,sha256=I5TOC1R1usWGyd4r3C9W_W6lbeKRVID1m5CaW6Vi4lM,18780
 tests/backends/test_kafka_commit_callback.py,sha256=svpY3T7FvhXvz2jp729e60LLn8MyFa8w88FQ1Y0Ursw,1987
 tests/backends/test_kafka_producer.py,sha256=LpwkqnstcCDxemlKZ0FpzNKrP-1UuXXY15P7P-spjhE,3912
 tests/backends/test_local.py,sha256=Mfd4DFuWVSVtl1GomQ6TIoWuJNcAliKqKU0BShPlEMY,3363
 tests/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/processing/test_processor.py,sha256=Gj86_WYQ-Eybz2YwLjNLonBfzJQQI9Vqh1WiYTbmcQk,23265
+tests/processing/test_processor.py,sha256=Ku_3z5aUeCRPRfeltHTSDLUMiATWmM_UXylhbE4kupg,25963
 tests/processing/strategies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/processing/strategies/test_all.py,sha256=ahAF-nbdmqVkYGNCg0OFCD6fzNTA-XxYrW8NQHajCDU,10167
 tests/processing/strategies/test_batching.py,sha256=nyyX0y6qYHX7jT4gCgsUjT5RzBMDrBp790SCmOizQ0Q,11787
@@ -75,7 +75,7 @@ tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/utils/test_concurrent.py,sha256=Gwdzym2UZ1HO3rhOSGmzxImWcLFygY8P7MXHT3Q0xTE,455
 tests/utils/test_metrics.py,sha256=bI0EtGgPokMQyEqX58i0-8zvLfxRP2nWaWr2wLMaJ_o,917
 tests/utils/test_retries.py,sha256=AxJLkXWeL9AjHv_p1n0pe8CXXJp24ZQIuYBHfNcmiz4,3075
-sentry_arroyo-2.32.3.dist-info/METADATA,sha256=EfJLX_0UhKzQRBftakhcDvPKZOqlR-JS17uEN-hNV4w,2208
-sentry_arroyo-2.32.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sentry_arroyo-2.32.3.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
-sentry_arroyo-2.32.3.dist-info/RECORD,,
+sentry_arroyo-2.32.5.dist-info/METADATA,sha256=Ld6VXfYDMWphFkcbBijnnHx4GebcIT91If8tqP_6Tho,2208
+sentry_arroyo-2.32.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sentry_arroyo-2.32.5.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
+sentry_arroyo-2.32.5.dist-info/RECORD,,

tests/backends/test_kafka.py

@@ -80,7 +80,7 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
     @property
     def configuration(self) -> KafkaBrokerConfig:
         config = {
-            "bootstrap.servers": os.environ.get("DEFAULT_BROKERS", "localhost:9092"),
+            "bootstrap.servers": os.environ.get("DEFAULT_BROKERS", "127.0.0.1:9092"),
         }
 
         return build_kafka_configuration(config)
@@ -231,6 +231,93 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
                 with pytest.raises(RuntimeError):
                     processor.run()
 
+    @mock.patch("arroyo.processing.processor.BACKPRESSURE_THRESHOLD", 0)
+    def test_assign_partition_during_pause(self) -> None:
+        if self.cooperative_sticky or self.kip_848:
+            pytest.skip("test does not work with cooperative-sticky rebalancing")
+
+        payloads = self.get_payloads()
+
+        strategy = mock.Mock()
+        strategy.submit.side_effect = MessageRejected()
+        factory = mock.Mock()
+        factory.create_with_partitions.return_value = strategy
+
+        partition_count = 2
+
+        with self.get_topic(partition_count) as topic, closing(
+            self.get_producer()
+        ) as producer, closing(
+            self.get_consumer(
+                "test_assign_partition_during_pause", enable_end_of_partition=True
+            )
+        ) as consumer_a, closing(
+            self.get_consumer(
+                "test_assign_partition_during_pause", enable_end_of_partition=True
+            )
+        ) as consumer_b:
+            for i in range(partition_count):
+                producer.produce(Partition(topic, i), next(payloads)).result(
+                    timeout=5.0
+                )
+
+            processor_a = StreamProcessor(
+                consumer_a, topic, factory, IMMEDIATE, handle_poll_while_paused=True
+            )
+
+            def wait_until_consumer_pauses(processor: StreamProcessor[Any]) -> None:
+                for _ in range(20):
+                    try:
+                        processor._run_once()
+                    except EndOfPartition:
+                        pass
+
+                    if processor._StreamProcessor__is_paused:  # type:ignore
+                        return
+                raise RuntimeError("processor was not paused")
+
+            # calling _run_once will pause both consumers because of the MessageRejected strategy above
+            wait_until_consumer_pauses(processor_a)
+
+            # consumer A has all the partitions
+            assert len(consumer_a.tell()) == 2
+            assert len(consumer_b.tell()) == 0
+
+            # consumer A has all partitions paused (both from consumer and from
+            # StreamProcessor POV)
+            assert consumer_a.paused()
+            assert processor_a._StreamProcessor__is_paused is True  # type:ignore
+
+            # subscribe with another consumer, now we should have rebalancing in the next few polls
+            processor_b = StreamProcessor(
+                consumer_b, topic, factory, IMMEDIATE, handle_poll_while_paused=True
+            )
+
+            for _ in range(10):
+                try:
+                    processor_a._run_once()
+                except EndOfPartition:
+                    pass
+                try:
+                    processor_b._run_once()
+                except EndOfPartition:
+                    pass
+
+            # balanced
+            assert len(consumer_a.tell()) == 1
+            assert len(consumer_b.tell()) == 1
+
+            # close B, but A has not polled yet, so it only has one partition still
+            consumer_b.close()
+            assert len(consumer_a.tell()) == 1
+
+            for _ in range(20):
+                try:
+                    processor_a._run_once()
+                except EndOfPartition:
+                    pass
+            assert len(consumer_a.tell()) == 2
+
     def test_consumer_polls_when_paused(self) -> None:
         strategy = mock.Mock()
         factory = mock.Mock()

tests/processing/test_processor.py

@@ -690,3 +690,75 @@ def test_processor_pause_with_invalid_message() -> None:
 
     processor._run_once()
     assert strategy.submit.call_args_list[-1] == mock.call(new_message)
+
+def test_processor_poll_while_paused() -> None:
+
+    topic = Topic("topic")
+
+    consumer = mock.Mock()
+    strategy = mock.Mock()
+    factory = mock.Mock()
+    factory.create_with_partitions.return_value = strategy
+
+    processor: StreamProcessor[int] = StreamProcessor(
+        consumer, topic, factory, IMMEDIATE, handle_poll_while_paused=True
+    )
+
+    # Subscribe to topic
+    subscribe_args, subscribe_kwargs = consumer.subscribe.call_args
+    assert subscribe_args[0] == [topic]
+
+    # Partition assignment
+    partition = Partition(topic, 0)
+    new_partition = Partition(topic, 1)
+    consumer.tell.return_value = {}
+    assignment_callback = subscribe_kwargs["on_assign"]
+    offsets = {partition: 0}
+    assignment_callback(offsets)
+
+    # Message that we will get from polling
+    message = Message(BrokerValue(0, partition, 0, datetime.now()))
+
+    # Message will be rejected
+    consumer.poll.return_value = message.value
+    strategy.submit.side_effect = MessageRejected()
+    with assert_changes(lambda: int(consumer.pause.call_count), 0, 1):
+        processor._run_once()
+        assert strategy.submit.call_args_list[-1] == mock.call(message)
+
+        with mock.patch("time.time", return_value=time.time() + 5):
+            processor._run_once()  # Should pause now
+
+    # Consumer is in paused state
+    # The same rejected message should be carried over
+
+    # All partitions are paused
+    consumer.paused.return_value = set(p for p in offsets)
+    # Simulate a continuous backpressure state where messages are being rejected
+    strategy.submit.side_effect = MessageRejected()
+
+    # Simulate Kafka returning nothing since the consumer is paused
+    consumer.poll.return_value = None
+
+    # The next poll returns nothing, but we are still carrying over the rejected message
+    processor._run_once()
+    assert consumer.poll.return_value is None
+
+    # At this point, let's say the message carried over is invalid (e.g. it could be stale)
+    strategy.submit.side_effect = InvalidMessage(partition, 0, needs_commit=False)
+
+    # Handles the invalid message and unpauses the consumer
+    with assert_changes(lambda: int(consumer.resume.call_count), 0, 1):
+        processor._run_once()
+
+    # Poll for the next message from Kafka, but this time the partition has changed
+    new_message = Message(BrokerValue(0, new_partition, 1, datetime.now()))
+    consumer.poll.return_value = new_message.value
+    processor._run_once()
+    assert processor._StreamProcessor__is_paused is False # type:ignore
+
+    strategy.submit.return_value = None
+    strategy.submit.side_effect = None
+
+    processor._run_once()
+    assert strategy.submit.call_args_list[-1] == mock.call(new_message)