sentry-arroyo 2.32.5__py3-none-any.whl → 2.33.1__py3-none-any.whl

arroyo/backends/kafka/configuration.py

@@ -237,8 +237,7 @@ def build_kafka_consumer_configuration(
     bootstrap_servers: Optional[Sequence[str]] = None,
     override_params: Optional[Mapping[str, Any]] = None,
     strict_offset_reset: Optional[bool] = None,
-    enable_auto_commit: bool = False,
-    retry_handle_destroyed: bool = False,
+    enable_auto_commit: bool = True,
 ) -> KafkaBrokerConfig:
 
     if auto_offset_reset is None:
@@ -264,8 +263,6 @@ def build_kafka_consumer_configuration(
         "arroyo.strict.offset.reset": strict_offset_reset,
         # this is an arroyo specific flag to enable auto-commit mode
         "arroyo.enable.auto.commit": enable_auto_commit,
-        # arroyo specific flag to enable retries when hitting `KafkaError._DESTROY` while committing
-        "arroyo.retry.broker.handle.destroyed": retry_handle_destroyed,
         # overridden to reduce memory usage when there's a large backlog
         "queued.max.messages.kbytes": queued_max_messages_kbytes,
         "queued.min.messages": queued_min_messages,

arroyo/backends/kafka/consumer.py

@@ -159,22 +159,13 @@ class KafkaConsumer(Consumer[KafkaPayload]):
     ) -> None:
         configuration = dict(configuration)
 
-        # Feature flag to enable retrying on `Broker handle destroyed` errors
-        # which can occur if we attempt to commit during a rebalance when
-        # the consumer group coordinator changed
-        self.__retry_handle_destroyed = as_kafka_configuration_bool(
-            configuration.pop("arroyo.retry.broker.handle.destroyed", False)
-        )
-
-        retryable_errors: Tuple[int, ...] = (
+        retryable_errors = (
             KafkaError.REQUEST_TIMED_OUT,
             KafkaError.NOT_COORDINATOR,
             KafkaError._WAIT_COORD,
             KafkaError.STALE_MEMBER_EPOCH,  # kip-848
             KafkaError.COORDINATOR_LOAD_IN_PROGRESS,
         )
-        if self.__retry_handle_destroyed:
-            retryable_errors += (KafkaError._DESTROY,)
 
         commit_retry_policy = BasicRetryPolicy(
             3,

arroyo/processing/processor.py

@@ -140,8 +140,6 @@ class StreamProcessor(Generic[TStrategyPayload]):
         commit_policy: CommitPolicy = ONCE_PER_SECOND,
         dlq_policy: Optional[DlqPolicy[TStrategyPayload]] = None,
         join_timeout: Optional[float] = None,
-        shutdown_strategy_before_consumer: bool = False,
-        handle_poll_while_paused: Optional[bool] = False,
     ) -> None:
         self.__consumer = consumer
         self.__processor_factory = processor_factory
@@ -166,9 +164,6 @@ class StreamProcessor(Generic[TStrategyPayload]):
         )
 
         self.__shutdown_requested = False
-        self.__shutdown_strategy_before_consumer = shutdown_strategy_before_consumer
-
-        self.__handle_poll_while_paused = handle_poll_while_paused
 
         # Buffers messages for DLQ. Messages are added when they are submitted for processing and
         # removed once the commit callback is fired as they are guaranteed to be valid at that point.
@@ -386,9 +381,12 @@ class StreamProcessor(Generic[TStrategyPayload]):
             start_dlq = time.time()
             invalid_message = self.__buffered_messages.pop(exc.partition, exc.offset)
             if invalid_message is None:
-                raise Exception(
-                    f"Invalid message not found in buffer {exc.partition} {exc.offset}",
-                ) from None
+                logger.error(
+                    "Invalid message not found in buffer %s %s",
+                    exc.partition,
+                    exc.offset,
+                )
+                return
 
             # XXX: This blocks if there are more than MAX_PENDING_FUTURES in the queue.
             try:
@@ -467,58 +465,19 @@ class StreamProcessor(Generic[TStrategyPayload]):
                         self.__is_paused = True
 
                     elif self.__is_paused:
-                        if self.__handle_poll_while_paused:
-                            maybe_message = self.__consumer.poll(0.1)
-                            if maybe_message is not None:
-                                # The paused consumer, in the above poll, has
-                                # gone through rebalancing. In this case we
-                                # expect that partition revocation cleared the
-                                # pause flag and the carried over message.
-                                # this assumption will not hold for cooperative-sticky rebalancing.
-                                assert (
-                                    not self.__is_paused
-                                ), "consumer unpaused itself without rebalancing"
-                                assert (
-                                    self.__message is None
-                                ), "consumer unpaused itself without rebalancing"
-                        # this path might raise AssertionErrors
-                        else:
-                            paused_partitions = set(self.__consumer.paused())
-                            all_partitions = set(self.__consumer.tell())
-                            unpaused_partitions = all_partitions - paused_partitions
-                            if unpaused_partitions:
-                                logger.warning(
-                                    "Processor in paused state while consumer is partially unpaused: %s, paused: %s",
-                                    unpaused_partitions,
-                                    paused_partitions,
-                                )
-                                self.__is_paused = False
-                                # unpause paused partitions... just in case a subset is paused
-                                self.__metrics_buffer.incr_counter(
-                                    "arroyo.consumer.resume", 1
-                                )
-                                self.__consumer.resume([*paused_partitions])
-                            else:
-                                # A paused consumer should still poll periodically to avoid it's partitions
-                                # getting revoked by the broker after reaching the max.poll.interval.ms
-                                # Polling a paused consumer should never yield a message.
-                                logger.warning(
-                                    "consumer.tell() value right before poll() is: %s",
-                                    self.__consumer.tell(),
-                                )
-                                maybe_message = self.__consumer.poll(0.1)
-                                if maybe_message is not None:
-                                    logger.warning(
-                                        "Received a message from partition: %s, \
-                                                    consumer.tell() value right after poll() is: %s \
-                                                    Some lines above consumer.tell() was called, all_partitons value was: %s \
-                                                    Some lines above consumer.paused() was called, paused_partitions value is: %s",
-                                        maybe_message.partition,
-                                        self.__consumer.tell(),
-                                        all_partitions,
-                                        paused_partitions,
-                                    )
-                                assert maybe_message is None
+                        maybe_message = self.__consumer.poll(0.1)
+                        if maybe_message is not None:
+                            # The paused consumer, in the above poll, has
+                            # gone through rebalancing. In this case we
+                            # expect that partition revocation cleared the
+                            # pause flag and the carried over message.
+                            # this assumption will not hold for cooperative-sticky rebalancing.
+                            assert (
+                                not self.__is_paused
+                            ), "consumer unpaused itself without rebalancing"
+                            assert (
+                                self.__message is None
+                            ), "consumer unpaused itself without rebalancing"
 
                     else:
                         time.sleep(0.01)
@@ -560,12 +519,11 @@ class StreamProcessor(Generic[TStrategyPayload]):
         self.__shutdown_requested = True
 
     def _shutdown(self) -> None:
-        # If shutdown_strategy_before_consumer is set, work around an issue
-        # where rdkafka would revoke our partition, but then also immediately
-        # revoke our member ID as well, causing join() of the CommitStrategy
-        # (that is running in the partition revocation callback) to crash.
-        if self.__shutdown_strategy_before_consumer:
-            self._close_processing_strategy()
+        # when we close() a consumer, rdkafka would would revoke our partition
+        # and call revocation callbacks, but also immediately revoke our member
+        # ID as well, causing join() of the CommitStrategy (that is running in
+        # the partition revocation callback) to crash.
+        self._close_processing_strategy()
 
         # close the consumer
         logger.info("Stopping consumer")

{sentry_arroyo-2.32.5.dist-info → sentry_arroyo-2.33.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.32.5
+Version: 2.33.1
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.32.5.dist-info → sentry_arroyo-2.33.1.dist-info}/RECORD

@@ -8,15 +8,15 @@ arroyo/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/backends/abstract.py,sha256=Wy9xhE1dtFiumG8Cz3JhksJ0rF74uJWZWq10UO1rxOI,9524
 arroyo/backends/kafka/__init__.py,sha256=xgf-AqHbQkJsh73YokO2uoyyHfZf8XwUp6BULtM8stI,445
 arroyo/backends/kafka/commit.py,sha256=LPsjvX5PPXR62DT6sa5GuSF78qk9F_L--Fz4kw7-m-s,3060
-arroyo/backends/kafka/configuration.py,sha256=42FQyrpIQGRaECBPljSaNO8E1RRyEoG1-a5cg3PwRe0,9356
-arroyo/backends/kafka/consumer.py,sha256=dzypkibGGieLg819TOoxtbVpVaxKYF9dazorQv5FxPg,34688
+arroyo/backends/kafka/configuration.py,sha256=voSOnySPlIjmd9LafpnfE_X4s2Bkxd6YTpdsTiZ2CNk,9140
+arroyo/backends/kafka/consumer.py,sha256=6exjpmSQxWVAdFCz52iIFQW3l3to-eJl9rCumY65e1o,34216
 arroyo/backends/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/backends/local/backend.py,sha256=hUXdCV6B5e7s4mjFC6HnIuUhjENU2tNZt5vuEOJmGZQ,13888
 arroyo/backends/local/storages/__init__.py,sha256=AGYujdAAcn3osoj9jq84IzTywYbkIDv9wRg2rLhLXeg,104
 arroyo/backends/local/storages/abstract.py,sha256=1qVQp6roxHkK6XT2aklZyZk1qq7RzcPN6Db_CA5--kg,2901
 arroyo/backends/local/storages/memory.py,sha256=AoKDsVZzBXkOJyWArKWp3vfGfU9xLlKFXE9gsJiMIzQ,2613
 arroyo/processing/__init__.py,sha256=vZVg0wJvJfoVzlzGvnL59bT6YNIRJNQ5t7oU045Qbk4,87
-arroyo/processing/processor.py,sha256=CHBvpAxH6T3MrUtoCJR7nTFJppqmIAx0DTosADrUAEM,23711
+arroyo/processing/processor.py,sha256=yEsA4WRhWoGJN9YVZVxOu52cIXEuriSbWV46HLNUX-I,20685
 arroyo/processing/strategies/__init__.py,sha256=EU_JMb54eOxMxaC5mIFpI-sAF-X2ZScbE8czBZ7bQkY,1106
 arroyo/processing/strategies/abstract.py,sha256=nu7juEz_aQmQIH35Z8u--FBuLjkK8_LQ1hIG2xpw9AA,4808
 arroyo/processing/strategies/batching.py,sha256=s89xC6lQpBseEaApu1iNTipXGKeO95OMwinj2VBKn9s,4778
@@ -46,17 +46,17 @@ examples/transform_and_produce/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 examples/transform_and_produce/batched.py,sha256=st2R6qTneAtV0JFbKP30Ti3sJDYj8Jkbmta9JckKdZU,2636
 examples/transform_and_produce/script.py,sha256=8kSMIjQNqGYEVyE0PvrfJh-a_UYCrJSstTp_De7kyyg,2306
 examples/transform_and_produce/simple.py,sha256=H7xqxItjl4tx34wVW5dy6mB9G39QucAtxkJSBzVmjgA,1637
-sentry_arroyo-2.32.5.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
+sentry_arroyo-2.33.1.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
 tests/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/backends/mixins.py,sha256=sfNyE0VTeiD3GHOnBYl-9urvPuURI2G1BWke0cz7Dvc,20445
 tests/backends/test_commit.py,sha256=iTHfK1qsBxim0XwxgMvNNSMqDUMEHoYkYBDcgxGBFbs,831
 tests/backends/test_confluent_producer.py,sha256=KWqgvjDvqAdd0HxngdWKsUJaV7Hl1L5vAVQhBYlHeHU,3146
-tests/backends/test_kafka.py,sha256=I5TOC1R1usWGyd4r3C9W_W6lbeKRVID1m5CaW6Vi4lM,18780
+tests/backends/test_kafka.py,sha256=YL6CAPW8EtAJpjMhY71tyqHwcKtVkruB9cRCj9xx0Dk,18658
 tests/backends/test_kafka_commit_callback.py,sha256=svpY3T7FvhXvz2jp729e60LLn8MyFa8w88FQ1Y0Ursw,1987
 tests/backends/test_kafka_producer.py,sha256=LpwkqnstcCDxemlKZ0FpzNKrP-1UuXXY15P7P-spjhE,3912
 tests/backends/test_local.py,sha256=Mfd4DFuWVSVtl1GomQ6TIoWuJNcAliKqKU0BShPlEMY,3363
 tests/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/processing/test_processor.py,sha256=Ku_3z5aUeCRPRfeltHTSDLUMiATWmM_UXylhbE4kupg,25963
+tests/processing/test_processor.py,sha256=f4WsNBo8RfNK2sPGWnwoJie1vfgaf74W1VAaazojwJ8,25934
 tests/processing/strategies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/processing/strategies/test_all.py,sha256=ahAF-nbdmqVkYGNCg0OFCD6fzNTA-XxYrW8NQHajCDU,10167
 tests/processing/strategies/test_batching.py,sha256=nyyX0y6qYHX7jT4gCgsUjT5RzBMDrBp790SCmOizQ0Q,11787
@@ -75,7 +75,7 @@ tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/utils/test_concurrent.py,sha256=Gwdzym2UZ1HO3rhOSGmzxImWcLFygY8P7MXHT3Q0xTE,455
 tests/utils/test_metrics.py,sha256=bI0EtGgPokMQyEqX58i0-8zvLfxRP2nWaWr2wLMaJ_o,917
 tests/utils/test_retries.py,sha256=AxJLkXWeL9AjHv_p1n0pe8CXXJp24ZQIuYBHfNcmiz4,3075
-sentry_arroyo-2.32.5.dist-info/METADATA,sha256=Ld6VXfYDMWphFkcbBijnnHx4GebcIT91If8tqP_6Tho,2208
-sentry_arroyo-2.32.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sentry_arroyo-2.32.5.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
-sentry_arroyo-2.32.5.dist-info/RECORD,,
+sentry_arroyo-2.33.1.dist-info/METADATA,sha256=3HGQ4TfTltxOTm50ADT6zxGKvp2arBF6MFjzk1Wt4cs,2208
+sentry_arroyo-2.33.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sentry_arroyo-2.33.1.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
+sentry_arroyo-2.33.1.dist-info/RECORD,,

tests/backends/test_kafka.py

@@ -261,9 +261,7 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
                     timeout=5.0
                 )
 
-            processor_a = StreamProcessor(
-                consumer_a, topic, factory, IMMEDIATE, handle_poll_while_paused=True
-            )
+            processor_a = StreamProcessor(consumer_a, topic, factory, IMMEDIATE)
 
             def wait_until_consumer_pauses(processor: StreamProcessor[Any]) -> None:
                 for _ in range(20):
@@ -289,9 +287,7 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
             assert processor_a._StreamProcessor__is_paused is True  # type:ignore
 
             # subscribe with another consumer, now we should have rebalancing in the next few polls
-            processor_b = StreamProcessor(
-                consumer_b, topic, factory, IMMEDIATE, handle_poll_while_paused=True
-            )
+            processor_b = StreamProcessor(consumer_b, topic, factory, IMMEDIATE)
 
             for _ in range(10):
                 try:

tests/processing/test_processor.py

@@ -691,6 +691,7 @@ def test_processor_pause_with_invalid_message() -> None:
     processor._run_once()
     assert strategy.submit.call_args_list[-1] == mock.call(new_message)
 
+
 def test_processor_poll_while_paused() -> None:
 
     topic = Topic("topic")
@@ -701,7 +702,7 @@ def test_processor_poll_while_paused() -> None:
     factory.create_with_partitions.return_value = strategy
 
     processor: StreamProcessor[int] = StreamProcessor(
-        consumer, topic, factory, IMMEDIATE, handle_poll_while_paused=True
+        consumer, topic, factory, IMMEDIATE
     )
 
     # Subscribe to topic
@@ -755,7 +756,7 @@ def test_processor_poll_while_paused() -> None:
     new_message = Message(BrokerValue(0, new_partition, 1, datetime.now()))
     consumer.poll.return_value = new_message.value
     processor._run_once()
-    assert processor._StreamProcessor__is_paused is False # type:ignore
+    assert processor._StreamProcessor__is_paused is False  # type:ignore
 
     strategy.submit.return_value = None
     strategy.submit.side_effect = None