sentry-arroyo 2.32.4__py3-none-any.whl → 2.33.0__py3-none-any.whl

arroyo/backends/kafka/configuration.py

@@ -237,8 +237,7 @@ def build_kafka_consumer_configuration(
     bootstrap_servers: Optional[Sequence[str]] = None,
     override_params: Optional[Mapping[str, Any]] = None,
     strict_offset_reset: Optional[bool] = None,
-    enable_auto_commit: bool = False,
-    retry_handle_destroyed: bool = False,
+    enable_auto_commit: bool = True,
 ) -> KafkaBrokerConfig:
 
     if auto_offset_reset is None:
@@ -264,8 +263,6 @@ def build_kafka_consumer_configuration(
         "arroyo.strict.offset.reset": strict_offset_reset,
         # this is an arroyo specific flag to enable auto-commit mode
         "arroyo.enable.auto.commit": enable_auto_commit,
-        # arroyo specific flag to enable retries when hitting `KafkaError._DESTROY` while committing
-        "arroyo.retry.broker.handle.destroyed": retry_handle_destroyed,
         # overridden to reduce memory usage when there's a large backlog
         "queued.max.messages.kbytes": queued_max_messages_kbytes,
         "queued.min.messages": queued_min_messages,

arroyo/backends/kafka/consumer.py

@@ -159,22 +159,13 @@ class KafkaConsumer(Consumer[KafkaPayload]):
     ) -> None:
         configuration = dict(configuration)
 
-        # Feature flag to enable retrying on `Broker handle destroyed` errors
-        # which can occur if we attempt to commit during a rebalance when
-        # the consumer group coordinator changed
-        self.__retry_handle_destroyed = as_kafka_configuration_bool(
-            configuration.pop("arroyo.retry.broker.handle.destroyed", False)
-        )
-
-        retryable_errors: Tuple[int, ...] = (
+        retryable_errors = (
             KafkaError.REQUEST_TIMED_OUT,
             KafkaError.NOT_COORDINATOR,
             KafkaError._WAIT_COORD,
             KafkaError.STALE_MEMBER_EPOCH,  # kip-848
             KafkaError.COORDINATOR_LOAD_IN_PROGRESS,
         )
-        if self.__retry_handle_destroyed:
-            retryable_errors += (KafkaError._DESTROY,)
 
         commit_retry_policy = BasicRetryPolicy(
             3,

arroyo/processing/processor.py

@@ -140,15 +140,14 @@ class StreamProcessor(Generic[TStrategyPayload]):
         commit_policy: CommitPolicy = ONCE_PER_SECOND,
         dlq_policy: Optional[DlqPolicy[TStrategyPayload]] = None,
         join_timeout: Optional[float] = None,
-        shutdown_strategy_before_consumer: bool = False,
     ) -> None:
         self.__consumer = consumer
         self.__processor_factory = processor_factory
         self.__metrics_buffer = MetricsBuffer()
 
-        self.__processing_strategy: Optional[ProcessingStrategy[TStrategyPayload]] = (
-            None
-        )
+        self.__processing_strategy: Optional[
+            ProcessingStrategy[TStrategyPayload]
+        ] = None
 
         self.__message: Optional[BrokerValue[TStrategyPayload]] = None
 
@@ -165,7 +164,6 @@ class StreamProcessor(Generic[TStrategyPayload]):
         )
 
         self.__shutdown_requested = False
-        self.__shutdown_strategy_before_consumer = shutdown_strategy_before_consumer
 
         # Buffers messages for DLQ. Messages are added when they are submitted for processing and
         # removed once the commit callback is fired as they are guaranteed to be valid at that point.
@@ -464,40 +462,20 @@ class StreamProcessor(Generic[TStrategyPayload]):
                         self.__is_paused = True
 
                     elif self.__is_paused:
-                        paused_partitions = set(self.__consumer.paused())
-                        all_partitions = set(self.__consumer.tell())
-                        unpaused_partitions = (
-                            all_partitions - paused_partitions
-                        )
-                        if unpaused_partitions:
-                            logger.warning(
-                                "Processor in paused state while consumer is partially unpaused: %s, paused: %s",
-                                unpaused_partitions,
-                                paused_partitions,
-                            )
-                            self.__is_paused = False
-                            # unpause paused partitions... just in case a subset is paused
-                            self.__metrics_buffer.incr_counter(
-                                "arroyo.consumer.resume", 1
-                            )
-                            self.__consumer.resume([*paused_partitions])
-                        else:
-                            # A paused consumer should still poll periodically to avoid it's partitions
-                            # getting revoked by the broker after reaching the max.poll.interval.ms
-                            # Polling a paused consumer should never yield a message.
-                            logger.warning("consumer.tell() value right before poll() is: %s", self.__consumer.tell())
-                            maybe_message = self.__consumer.poll(0.1)
-                            if maybe_message is not None:
-                                logger.warning("Received a message from partition: %s, \
-                                                consumer.tell() value right after poll() is: %s \
-                                                Some lines above consumer.tell() was called, all_partitons value was: %s \
-                                                Some lines above consumer.paused() was called, paused_partitions value is: %s",
-                                                maybe_message.partition,
-                                                self.__consumer.tell(),
-                                                all_partitions,
-                                                paused_partitions
-                                                )
-                            assert maybe_message is None
+                        maybe_message = self.__consumer.poll(0.1)
+                        if maybe_message is not None:
+                            # The paused consumer, in the above poll, has
+                            # gone through rebalancing. In this case we
+                            # expect that partition revocation cleared the
+                            # pause flag and the carried over message.
+                            # this assumption will not hold for cooperative-sticky rebalancing.
+                            assert (
+                                not self.__is_paused
+                            ), "consumer unpaused itself without rebalancing"
+                            assert (
+                                self.__message is None
+                            ), "consumer unpaused itself without rebalancing"
+
                     else:
                         time.sleep(0.01)
 
@@ -538,12 +516,11 @@ class StreamProcessor(Generic[TStrategyPayload]):
         self.__shutdown_requested = True
 
     def _shutdown(self) -> None:
-        # If shutdown_strategy_before_consumer is set, work around an issue
-        # where rdkafka would revoke our partition, but then also immediately
-        # revoke our member ID as well, causing join() of the CommitStrategy
-        # (that is running in the partition revocation callback) to crash.
-        if self.__shutdown_strategy_before_consumer:
-            self._close_processing_strategy()
+        # when we close() a consumer, rdkafka would would revoke our partition
+        # and call revocation callbacks, but also immediately revoke our member
+        # ID as well, causing join() of the CommitStrategy (that is running in
+        # the partition revocation callback) to crash.
+        self._close_processing_strategy()
 
         # close the consumer
         logger.info("Stopping consumer")

{sentry_arroyo-2.32.4.dist-info → sentry_arroyo-2.33.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.32.4
+Version: 2.33.0
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.32.4.dist-info → sentry_arroyo-2.33.0.dist-info}/RECORD

@@ -8,15 +8,15 @@ arroyo/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/backends/abstract.py,sha256=Wy9xhE1dtFiumG8Cz3JhksJ0rF74uJWZWq10UO1rxOI,9524
 arroyo/backends/kafka/__init__.py,sha256=xgf-AqHbQkJsh73YokO2uoyyHfZf8XwUp6BULtM8stI,445
 arroyo/backends/kafka/commit.py,sha256=LPsjvX5PPXR62DT6sa5GuSF78qk9F_L--Fz4kw7-m-s,3060
-arroyo/backends/kafka/configuration.py,sha256=42FQyrpIQGRaECBPljSaNO8E1RRyEoG1-a5cg3PwRe0,9356
-arroyo/backends/kafka/consumer.py,sha256=dzypkibGGieLg819TOoxtbVpVaxKYF9dazorQv5FxPg,34688
+arroyo/backends/kafka/configuration.py,sha256=voSOnySPlIjmd9LafpnfE_X4s2Bkxd6YTpdsTiZ2CNk,9140
+arroyo/backends/kafka/consumer.py,sha256=6exjpmSQxWVAdFCz52iIFQW3l3to-eJl9rCumY65e1o,34216
 arroyo/backends/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/backends/local/backend.py,sha256=hUXdCV6B5e7s4mjFC6HnIuUhjENU2tNZt5vuEOJmGZQ,13888
 arroyo/backends/local/storages/__init__.py,sha256=AGYujdAAcn3osoj9jq84IzTywYbkIDv9wRg2rLhLXeg,104
 arroyo/backends/local/storages/abstract.py,sha256=1qVQp6roxHkK6XT2aklZyZk1qq7RzcPN6Db_CA5--kg,2901
 arroyo/backends/local/storages/memory.py,sha256=AoKDsVZzBXkOJyWArKWp3vfGfU9xLlKFXE9gsJiMIzQ,2613
 arroyo/processing/__init__.py,sha256=vZVg0wJvJfoVzlzGvnL59bT6YNIRJNQ5t7oU045Qbk4,87
-arroyo/processing/processor.py,sha256=raPgmhl9K9kwI9ZlhCZELJGaba3xK525gmTi4bABenc,22377
+arroyo/processing/processor.py,sha256=l8nGwRpVe5JWZ-zA6KTrrsJezbjKvfKvsDHBrtUtTLw,20632
 arroyo/processing/strategies/__init__.py,sha256=EU_JMb54eOxMxaC5mIFpI-sAF-X2ZScbE8czBZ7bQkY,1106
 arroyo/processing/strategies/abstract.py,sha256=nu7juEz_aQmQIH35Z8u--FBuLjkK8_LQ1hIG2xpw9AA,4808
 arroyo/processing/strategies/batching.py,sha256=s89xC6lQpBseEaApu1iNTipXGKeO95OMwinj2VBKn9s,4778
@@ -46,17 +46,17 @@ examples/transform_and_produce/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 examples/transform_and_produce/batched.py,sha256=st2R6qTneAtV0JFbKP30Ti3sJDYj8Jkbmta9JckKdZU,2636
 examples/transform_and_produce/script.py,sha256=8kSMIjQNqGYEVyE0PvrfJh-a_UYCrJSstTp_De7kyyg,2306
 examples/transform_and_produce/simple.py,sha256=H7xqxItjl4tx34wVW5dy6mB9G39QucAtxkJSBzVmjgA,1637
-sentry_arroyo-2.32.4.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
+sentry_arroyo-2.33.0.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
 tests/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/backends/mixins.py,sha256=sfNyE0VTeiD3GHOnBYl-9urvPuURI2G1BWke0cz7Dvc,20445
 tests/backends/test_commit.py,sha256=iTHfK1qsBxim0XwxgMvNNSMqDUMEHoYkYBDcgxGBFbs,831
 tests/backends/test_confluent_producer.py,sha256=KWqgvjDvqAdd0HxngdWKsUJaV7Hl1L5vAVQhBYlHeHU,3146
-tests/backends/test_kafka.py,sha256=wBFCKEHoP6h0uG1bgDuzk84IZmrV_UVOFCrtbxztmJg,15506
+tests/backends/test_kafka.py,sha256=YL6CAPW8EtAJpjMhY71tyqHwcKtVkruB9cRCj9xx0Dk,18658
 tests/backends/test_kafka_commit_callback.py,sha256=svpY3T7FvhXvz2jp729e60LLn8MyFa8w88FQ1Y0Ursw,1987
 tests/backends/test_kafka_producer.py,sha256=LpwkqnstcCDxemlKZ0FpzNKrP-1UuXXY15P7P-spjhE,3912
 tests/backends/test_local.py,sha256=Mfd4DFuWVSVtl1GomQ6TIoWuJNcAliKqKU0BShPlEMY,3363
 tests/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/processing/test_processor.py,sha256=Gj86_WYQ-Eybz2YwLjNLonBfzJQQI9Vqh1WiYTbmcQk,23265
+tests/processing/test_processor.py,sha256=f4WsNBo8RfNK2sPGWnwoJie1vfgaf74W1VAaazojwJ8,25934
 tests/processing/strategies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/processing/strategies/test_all.py,sha256=ahAF-nbdmqVkYGNCg0OFCD6fzNTA-XxYrW8NQHajCDU,10167
 tests/processing/strategies/test_batching.py,sha256=nyyX0y6qYHX7jT4gCgsUjT5RzBMDrBp790SCmOizQ0Q,11787
@@ -75,7 +75,7 @@ tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/utils/test_concurrent.py,sha256=Gwdzym2UZ1HO3rhOSGmzxImWcLFygY8P7MXHT3Q0xTE,455
 tests/utils/test_metrics.py,sha256=bI0EtGgPokMQyEqX58i0-8zvLfxRP2nWaWr2wLMaJ_o,917
 tests/utils/test_retries.py,sha256=AxJLkXWeL9AjHv_p1n0pe8CXXJp24ZQIuYBHfNcmiz4,3075
-sentry_arroyo-2.32.4.dist-info/METADATA,sha256=pvsMvCmDOq3xqVGXTolHjmZhwwER_sVRLFBSuFpV-ms,2208
-sentry_arroyo-2.32.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sentry_arroyo-2.32.4.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
-sentry_arroyo-2.32.4.dist-info/RECORD,,
+sentry_arroyo-2.33.0.dist-info/METADATA,sha256=ycSuuLcy8YTgaDmntDbj5HWy9iFaIKweYmeqWUTpiCk,2208
+sentry_arroyo-2.33.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sentry_arroyo-2.33.0.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
+sentry_arroyo-2.33.0.dist-info/RECORD,,

tests/backends/test_kafka.py

@@ -80,7 +80,7 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
     @property
     def configuration(self) -> KafkaBrokerConfig:
         config = {
-            "bootstrap.servers": os.environ.get("DEFAULT_BROKERS", "localhost:9092"),
+            "bootstrap.servers": os.environ.get("DEFAULT_BROKERS", "127.0.0.1:9092"),
         }
 
         return build_kafka_configuration(config)
@@ -231,6 +231,89 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
                 with pytest.raises(RuntimeError):
                     processor.run()
 
+    @mock.patch("arroyo.processing.processor.BACKPRESSURE_THRESHOLD", 0)
+    def test_assign_partition_during_pause(self) -> None:
+        if self.cooperative_sticky or self.kip_848:
+            pytest.skip("test does not work with cooperative-sticky rebalancing")
+
+        payloads = self.get_payloads()
+
+        strategy = mock.Mock()
+        strategy.submit.side_effect = MessageRejected()
+        factory = mock.Mock()
+        factory.create_with_partitions.return_value = strategy
+
+        partition_count = 2
+
+        with self.get_topic(partition_count) as topic, closing(
+            self.get_producer()
+        ) as producer, closing(
+            self.get_consumer(
+                "test_assign_partition_during_pause", enable_end_of_partition=True
+            )
+        ) as consumer_a, closing(
+            self.get_consumer(
+                "test_assign_partition_during_pause", enable_end_of_partition=True
+            )
+        ) as consumer_b:
+            for i in range(partition_count):
+                producer.produce(Partition(topic, i), next(payloads)).result(
+                    timeout=5.0
+                )
+
+            processor_a = StreamProcessor(consumer_a, topic, factory, IMMEDIATE)
+
+            def wait_until_consumer_pauses(processor: StreamProcessor[Any]) -> None:
+                for _ in range(20):
+                    try:
+                        processor._run_once()
+                    except EndOfPartition:
+                        pass
+
+                    if processor._StreamProcessor__is_paused:  # type:ignore
+                        return
+                raise RuntimeError("processor was not paused")
+
+            # calling _run_once will pause both consumers because of the MessageRejected strategy above
+            wait_until_consumer_pauses(processor_a)
+
+            # consumer A has all the partitions
+            assert len(consumer_a.tell()) == 2
+            assert len(consumer_b.tell()) == 0
+
+            # consumer A has all partitions paused (both from consumer and from
+            # StreamProcessor POV)
+            assert consumer_a.paused()
+            assert processor_a._StreamProcessor__is_paused is True  # type:ignore
+
+            # subscribe with another consumer, now we should have rebalancing in the next few polls
+            processor_b = StreamProcessor(consumer_b, topic, factory, IMMEDIATE)
+
+            for _ in range(10):
+                try:
+                    processor_a._run_once()
+                except EndOfPartition:
+                    pass
+                try:
+                    processor_b._run_once()
+                except EndOfPartition:
+                    pass
+
+            # balanced
+            assert len(consumer_a.tell()) == 1
+            assert len(consumer_b.tell()) == 1
+
+            # close B, but A has not polled yet, so it only has one partition still
+            consumer_b.close()
+            assert len(consumer_a.tell()) == 1
+
+            for _ in range(20):
+                try:
+                    processor_a._run_once()
+                except EndOfPartition:
+                    pass
+            assert len(consumer_a.tell()) == 2
+
     def test_consumer_polls_when_paused(self) -> None:
         strategy = mock.Mock()
         factory = mock.Mock()

tests/processing/test_processor.py

@@ -690,3 +690,76 @@ def test_processor_pause_with_invalid_message() -> None:
 
     processor._run_once()
     assert strategy.submit.call_args_list[-1] == mock.call(new_message)
+
+
+def test_processor_poll_while_paused() -> None:
+
+    topic = Topic("topic")
+
+    consumer = mock.Mock()
+    strategy = mock.Mock()
+    factory = mock.Mock()
+    factory.create_with_partitions.return_value = strategy
+
+    processor: StreamProcessor[int] = StreamProcessor(
+        consumer, topic, factory, IMMEDIATE
+    )
+
+    # Subscribe to topic
+    subscribe_args, subscribe_kwargs = consumer.subscribe.call_args
+    assert subscribe_args[0] == [topic]
+
+    # Partition assignment
+    partition = Partition(topic, 0)
+    new_partition = Partition(topic, 1)
+    consumer.tell.return_value = {}
+    assignment_callback = subscribe_kwargs["on_assign"]
+    offsets = {partition: 0}
+    assignment_callback(offsets)
+
+    # Message that we will get from polling
+    message = Message(BrokerValue(0, partition, 0, datetime.now()))
+
+    # Message will be rejected
+    consumer.poll.return_value = message.value
+    strategy.submit.side_effect = MessageRejected()
+    with assert_changes(lambda: int(consumer.pause.call_count), 0, 1):
+        processor._run_once()
+        assert strategy.submit.call_args_list[-1] == mock.call(message)
+
+        with mock.patch("time.time", return_value=time.time() + 5):
+            processor._run_once()  # Should pause now
+
+    # Consumer is in paused state
+    # The same rejected message should be carried over
+
+    # All partitions are paused
+    consumer.paused.return_value = set(p for p in offsets)
+    # Simulate a continuous backpressure state where messages are being rejected
+    strategy.submit.side_effect = MessageRejected()
+
+    # Simulate Kafka returning nothing since the consumer is paused
+    consumer.poll.return_value = None
+
+    # The next poll returns nothing, but we are still carrying over the rejected message
+    processor._run_once()
+    assert consumer.poll.return_value is None
+
+    # At this point, let's say the message carried over is invalid (e.g. it could be stale)
+    strategy.submit.side_effect = InvalidMessage(partition, 0, needs_commit=False)
+
+    # Handles the invalid message and unpauses the consumer
+    with assert_changes(lambda: int(consumer.resume.call_count), 0, 1):
+        processor._run_once()
+
+    # Poll for the next message from Kafka, but this time the partition has changed
+    new_message = Message(BrokerValue(0, new_partition, 1, datetime.now()))
+    consumer.poll.return_value = new_message.value
+    processor._run_once()
+    assert processor._StreamProcessor__is_paused is False  # type:ignore
+
+    strategy.submit.return_value = None
+    strategy.submit.side_effect = None
+
+    processor._run_once()
+    assert strategy.submit.call_args_list[-1] == mock.call(new_message)