sentry-arroyo 2.19.9__py3-none-any.whl → 2.19.10__py3-none-any.whl

arroyo/backends/kafka/consumer.py

@@ -157,13 +157,13 @@ class KafkaConsumer(Consumer[KafkaPayload]):
                 KafkaError.REQUEST_TIMED_OUT,
                 KafkaError.NOT_COORDINATOR,
                 KafkaError._WAIT_COORD,
+                KafkaError.STALE_MEMBER_EPOCH,  # kip-848
             ),
         )
 
         configuration = dict(configuration)
-        self.__is_incremental = (
+        self.__is_cooperative_sticky = (
             configuration.get("partition.assignment.strategy") == "cooperative-sticky"
-            or configuration.get("group.protocol") == "consumer"
         )
         auto_offset_reset = configuration.get("auto.offset.reset", "largest")
 
@@ -463,7 +463,7 @@ class KafkaConsumer(Consumer[KafkaPayload]):
             ConfluentTopicPartition(partition.topic.name, partition.index, offset)
             for partition, offset in offsets.items()
         ]
-        if self.__is_incremental:
+        if self.__is_cooperative_sticky:
             self.__consumer.incremental_assign(partitions)
         else:
             self.__consumer.assign(partitions)

arroyo/backends/local/backend.py

@@ -38,9 +38,9 @@ class LocalBroker(Generic[TStrategyPayload]):
         self.__message_storage = message_storage
         self.__clock = clock
 
-        self.__offsets: MutableMapping[
-            str, MutableMapping[Partition, int]
-        ] = defaultdict(dict)
+        self.__offsets: MutableMapping[str, MutableMapping[Partition, int]] = (
+            defaultdict(dict)
+        )
 
         # The active subscriptions are stored by consumer group as a mapping
         # between the consumer and it's subscribed topics.

arroyo/dlq.py

@@ -49,7 +49,7 @@ class InvalidMessage(Exception):
         offset: int,
         needs_commit: bool = True,
         reason: Optional[str] = None,
-        log_exception: bool = True
+        log_exception: bool = True,
     ) -> None:
         self.partition = partition
         self.offset = offset
@@ -208,7 +208,9 @@ class NoopDlqProducer(DlqProducer[Any]):
     """
 
     def produce(
-        self, value: BrokerValue[KafkaPayload], reason: Optional[str] = None,
+        self,
+        value: BrokerValue[KafkaPayload],
+        reason: Optional[str] = None,
     ) -> Future[BrokerValue[KafkaPayload]]:
         future: Future[BrokerValue[KafkaPayload]] = Future()
         future.set_running_or_notify_cancel()
@@ -340,7 +342,7 @@ class DlqPolicyWrapper(Generic[TStrategyPayload]):
         self,
         policy: DlqPolicy[TStrategyPayload],
     ) -> None:
-        self.MAX_PENDING_FUTURES = 1000  # This is a per partition max
+        self.MAX_PENDING_FUTURES = 2000  # This is a per partition max
         self.__dlq_policy = policy
 
         self.__futures: MutableMapping[
@@ -362,7 +364,9 @@ class DlqPolicyWrapper(Generic[TStrategyPayload]):
             self.__dlq_policy.limit, assignment
         )
 
-    def produce(self, message: BrokerValue[TStrategyPayload], reason: Optional[str] = None) -> None:
+    def produce(
+        self, message: BrokerValue[TStrategyPayload], reason: Optional[str] = None
+    ) -> None:
         """
         Removes all completed futures, then appends the given future to the list.
         Blocks if the list is full. If the DLQ limit is exceeded, an exception is raised.

arroyo/processing/processor.py

@@ -143,9 +143,9 @@ class StreamProcessor(Generic[TStrategyPayload]):
         self.__processor_factory = processor_factory
         self.__metrics_buffer = MetricsBuffer()
 
-        self.__processing_strategy: Optional[
-            ProcessingStrategy[TStrategyPayload]
-        ] = None
+        self.__processing_strategy: Optional[ProcessingStrategy[TStrategyPayload]] = (
+            None
+        )
 
         self.__message: Optional[BrokerValue[TStrategyPayload]] = None
 

arroyo/processing/strategies/filter.py

@@ -55,9 +55,9 @@ class FilterStep(ProcessingStrategy[Union[FilteredPayload, TStrategyPayload]]):
         self.__next_step = next_step
 
         if commit_policy is not None:
-            self.__commit_policy_state: Optional[
-                CommitPolicyState
-            ] = commit_policy.get_state_machine()
+            self.__commit_policy_state: Optional[CommitPolicyState] = (
+                commit_policy.get_state_machine()
+            )
         else:
             self.__commit_policy_state = None
 

arroyo/processing/strategies/run_task_with_multiprocessing.py

@@ -625,9 +625,9 @@ class RunTaskWithMultiprocessing(
         while self.__processes:
             try:
                 self.__check_for_results_impl(
-                    timeout=max(deadline - time.time(), 0)
-                    if deadline is not None
-                    else None
+                    timeout=(
+                        max(deadline - time.time(), 0) if deadline is not None else None
+                    )
                 )
             except NextStepTimeoutError:
                 if deadline is None or deadline > time.time():

{sentry_arroyo-2.19.9.dist-info → sentry_arroyo-2.19.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: sentry-arroyo
-Version: 2.19.9
+Version: 2.19.10
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry
@@ -13,7 +13,16 @@ Classifier: Programming Language :: Python
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: confluent-kafka>=2.3.0
+Requires-Dist: confluent-kafka>=2.7.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary
 
 # Arroyo
 

{sentry_arroyo-2.19.9.dist-info → sentry_arroyo-2.19.10.dist-info}/RECORD

@@ -1,6 +1,6 @@
 arroyo/__init__.py,sha256=fcpHZd2P3MxWl6PJJ8n__fM_NRIfiUE8tKN-orv6lb0,187
 arroyo/commit.py,sha256=oFihWUW8fLsjomWh0o085qIHe9vwVNgoOJC6JQdFM7M,2235
-arroyo/dlq.py,sha256=BDmtJXrUKc8Pff7_AkRLE7accyyDkzakHe3mRPGJkNk,15631
+arroyo/dlq.py,sha256=-mSckVjUZ2j1UgRcxsF5App78-5xb25EaW0zmHDMUH4,15662
 arroyo/errors.py,sha256=IbtoIbz_m5QrxNRBLOxiy-hOfJQTEwNPCyq6yqedJYk,1059
 arroyo/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/types.py,sha256=sLY0x030np4UmbaW5C1KH1se7Z2pjQiPvAe5x2sXf7A,5684
@@ -9,20 +9,20 @@ arroyo/backends/abstract.py,sha256=PicUWWsgH-41XpHDcuq86P4PpPx7GWgLWdwnw4B0obo,7
 arroyo/backends/kafka/__init__.py,sha256=TZ0omd3LFXcZUaN_soFTuTgbjNEQYF1mF_i6_KIfCNo,306
 arroyo/backends/kafka/commit.py,sha256=LPsjvX5PPXR62DT6sa5GuSF78qk9F_L--Fz4kw7-m-s,3060
 arroyo/backends/kafka/configuration.py,sha256=D9zre9H2xagUsk7qBA-bm38V3_4Mg_X5hpKsKx2BkM8,3048
-arroyo/backends/kafka/consumer.py,sha256=sWqmn8jDf-wfZOXRc7vrotEepfeRbtdpzoP46VDf49A,27721
+arroyo/backends/kafka/consumer.py,sha256=fo5a0tCq0XFvMA9drVCUCBMAXuA_x5KlpBU4G2LsNIo,27728
 arroyo/backends/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arroyo/backends/local/backend.py,sha256=pQeZRUo9KDWlOT1pICWM0MvS39QuJqHnN5ryrmcxnSc,13878
+arroyo/backends/local/backend.py,sha256=7odjCnzoGgdo8JHLgG1ntaXa-ZR9GteGkquiA2WAWmM,13880
 arroyo/backends/local/storages/__init__.py,sha256=AGYujdAAcn3osoj9jq84IzTywYbkIDv9wRg2rLhLXeg,104
 arroyo/backends/local/storages/abstract.py,sha256=1qVQp6roxHkK6XT2aklZyZk1qq7RzcPN6Db_CA5--kg,2901
 arroyo/backends/local/storages/memory.py,sha256=AoKDsVZzBXkOJyWArKWp3vfGfU9xLlKFXE9gsJiMIzQ,2613
 arroyo/processing/__init__.py,sha256=vZVg0wJvJfoVzlzGvnL59bT6YNIRJNQ5t7oU045Qbk4,87
-arroyo/processing/processor.py,sha256=RRLp8qcEL04TSvHaGuF0nc6RQ-HOiBzzI8nUwV5hZ2M,20025
+arroyo/processing/processor.py,sha256=iTHQ8OI3rp6Pi43AWfnuAHC91eq9ZAp_i61-zsB1dq8,20027
 arroyo/processing/strategies/__init__.py,sha256=EU_JMb54eOxMxaC5mIFpI-sAF-X2ZScbE8czBZ7bQkY,1106
 arroyo/processing/strategies/abstract.py,sha256=nu7juEz_aQmQIH35Z8u--FBuLjkK8_LQ1hIG2xpw9AA,4808
 arroyo/processing/strategies/batching.py,sha256=s89xC6lQpBseEaApu1iNTipXGKeO95OMwinj2VBKn9s,4778
 arroyo/processing/strategies/buffer.py,sha256=7s-tmXOMKlmvElZ6GKLOTjqTAfrrjS7AU_4b_kDHi7s,7066
 arroyo/processing/strategies/commit.py,sha256=F5QtYHA4nLeNWHCLFlVQTNCw2PpgTzX8HDDmhs-voLM,1413
-arroyo/processing/strategies/filter.py,sha256=RxR4wclEp-VfrgNfWe4TCTHzw33TMKJi5GaeHva7sqs,4871
+arroyo/processing/strategies/filter.py,sha256=dzx9BaIVigxFGfsrNYAWJYTDCanIp3RiV91_dNHF5cc,4873
 arroyo/processing/strategies/guard.py,sha256=oC3XhyvYUg5CvSosUTXrCw9iA_hmN6EH2Lr-lpeOkI8,4875
 arroyo/processing/strategies/healthcheck.py,sha256=fEVB516nZVZWvZLzi4_ImaE6F7QQQJIymW6HgiUYUSA,1775
 arroyo/processing/strategies/noop.py,sha256=BtWkC93QpZCKAB4qTsXLaqrYsllE1Olp6mY2YeuK9ag,726
@@ -30,7 +30,7 @@ arroyo/processing/strategies/produce.py,sha256=v1mJv5rbyXal7ESIfsihbTGW-vZ0_niJn
 arroyo/processing/strategies/reduce.py,sha256=xv9bYisgHHyS8fVD1PdGi4TJsaK-4RAhMEDh4WHhYfI,3933
 arroyo/processing/strategies/run_task.py,sha256=MGe2UcIWN7FkPc9plKzRVUNbZ7Sk0jWjw1z2vVOFI_I,2160
 arroyo/processing/strategies/run_task_in_threads.py,sha256=f1sb2AG-BLz11X78jfhtERIkdFogrV8vtdT3pyJdkx0,6144
-arroyo/processing/strategies/run_task_with_multiprocessing.py,sha256=XhLRiMXFhZTRlxCTTmeI6SSTxy9SwiDHHiqq6mow-Z0,34409
+arroyo/processing/strategies/run_task_with_multiprocessing.py,sha256=jNb_e3hcXDlnLE0XCgVjbLHXrob8V53aPGeMWtu4ToI,34417
 arroyo/processing/strategies/unfold.py,sha256=wZDNdMo1Ln27P1tUvLO2svhL-JDs2ZCyIFF2cq0DxVQ,3400
 arroyo/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/utils/clock.py,sha256=r2EMO4nL5qIb1xnAd1sTAk2yK1UltyUi04lk5BqWKIc,944
@@ -47,9 +47,9 @@ examples/transform_and_produce/batched.py,sha256=st2R6qTneAtV0JFbKP30Ti3sJDYj8Jk
 examples/transform_and_produce/script.py,sha256=8kSMIjQNqGYEVyE0PvrfJh-a_UYCrJSstTp_De7kyyg,2306
 examples/transform_and_produce/simple.py,sha256=H7xqxItjl4tx34wVW5dy6mB9G39QucAtxkJSBzVmjgA,1637
 tests/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/backends/mixins.py,sha256=31t7B4a8cJ65XeUzv4ZoG9aPU1lTpDZXgkQrEyKhyqE,20131
+tests/backends/mixins.py,sha256=sfNyE0VTeiD3GHOnBYl-9urvPuURI2G1BWke0cz7Dvc,20445
 tests/backends/test_commit.py,sha256=iTHfK1qsBxim0XwxgMvNNSMqDUMEHoYkYBDcgxGBFbs,831
-tests/backends/test_kafka.py,sha256=cB_GIY1yWT9w2VB1PWrvTr0lCa46r5uDfxisDFrcSmg,12089
+tests/backends/test_kafka.py,sha256=mNwisb5PQPlBs5TIZtk6hL_eSbwAdwZ0RDD05WxDYps,12188
 tests/backends/test_local.py,sha256=Mfd4DFuWVSVtl1GomQ6TIoWuJNcAliKqKU0BShPlEMY,3363
 tests/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/processing/test_processor.py,sha256=PDCrmhWAt_wZKwYlBzDuEGsd8PnjKM-p4ySaPvVd11k,20781
@@ -71,8 +71,8 @@ tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/utils/test_concurrent.py,sha256=Gwdzym2UZ1HO3rhOSGmzxImWcLFygY8P7MXHT3Q0xTE,455
 tests/utils/test_metrics.py,sha256=bI0EtGgPokMQyEqX58i0-8zvLfxRP2nWaWr2wLMaJ_o,917
 tests/utils/test_retries.py,sha256=AxJLkXWeL9AjHv_p1n0pe8CXXJp24ZQIuYBHfNcmiz4,3075
-sentry_arroyo-2.19.9.dist-info/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
-sentry_arroyo-2.19.9.dist-info/METADATA,sha256=wmVhRiE5W4tvuCT3CCuAYsdi6q0ZjTPq8NxhA7zCuKA,1989
-sentry_arroyo-2.19.9.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-sentry_arroyo-2.19.9.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
-sentry_arroyo-2.19.9.dist-info/RECORD,,
+sentry_arroyo-2.19.10.dist-info/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
+sentry_arroyo-2.19.10.dist-info/METADATA,sha256=GS3IuMT7G1FFmRH6I3qVPW6GX1BtmN3uX9ji3XxYF70,2179
+sentry_arroyo-2.19.10.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+sentry_arroyo-2.19.10.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
+sentry_arroyo-2.19.10.dist-info/RECORD,,

{sentry_arroyo-2.19.9.dist-info → sentry_arroyo-2.19.10.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

tests/backends/mixins.py

@@ -15,6 +15,7 @@ from tests.assertions import assert_changes, assert_does_not_change
 
 class StreamsTestMixin(ABC, Generic[TStrategyPayload]):
     cooperative_sticky = False
+    kip_848 = False
 
     @abstractmethod
     def get_topic(self, partitions: int = 1) -> ContextManager[Topic]:
@@ -421,7 +422,7 @@ class StreamsTestMixin(ABC, Generic[TStrategyPayload]):
             def wait_until_rebalancing(
                 from_consumer: Consumer[Any], to_consumer: Consumer[Any]
             ) -> None:
-                for _ in range(10):
+                for _ in range(20):
                     assert from_consumer.poll(0) is None
                     if to_consumer.poll(1.0) is not None:
                         return
@@ -453,9 +454,10 @@ class StreamsTestMixin(ABC, Generic[TStrategyPayload]):
 
             wait_until_rebalancing(consumer_a, consumer_b)
 
-            if self.cooperative_sticky:
+            if self.cooperative_sticky or self.kip_848:
                 # within incremental rebalancing, only one partition should have been reassigned to the consumer_b, and consumer_a should remain paused
-                assert consumer_a.paused() == [Partition(topic, 1)]
+                # Either partition 0 or 1 might be the paused one
+                assert len(consumer_a.paused()) == 1
                 assert consumer_a.poll(10.0) is None
             else:
                 # The first consumer should have had its offsets rolled back, as
@@ -481,20 +483,28 @@ class StreamsTestMixin(ABC, Generic[TStrategyPayload]):
 
             assert len(consumer_b.tell()) == 2
 
-            if self.cooperative_sticky:
+            if self.cooperative_sticky or self.kip_848:
+                consumer_a_on_assign.assert_has_calls(
+                    [
+                        mock.call({Partition(topic, 0): 0, Partition(topic, 1): 0}),
+                    ]
+                )
 
-                assert consumer_a_on_assign.mock_calls == [
-                    mock.call({Partition(topic, 0): 0, Partition(topic, 1): 0}),
-                ]
-                assert consumer_a_on_revoke.mock_calls == [
-                    mock.call([Partition(topic, 0)]),
-                    mock.call([Partition(topic, 1)]),
-                ]
+                consumer_a_on_revoke.assert_has_calls(
+                    [
+                        mock.call([Partition(topic, 0)]),
+                        mock.call([Partition(topic, 1)]),
+                    ],
+                    any_order=True,
+                )
 
-                assert consumer_b_on_assign.mock_calls == [
-                    mock.call({Partition(topic, 0): 0}),
-                    mock.call({Partition(topic, 1): 0}),
-                ]
+                consumer_b_on_assign.assert_has_calls(
+                    [
+                        mock.call({Partition(topic, 0): 0}),
+                        mock.call({Partition(topic, 1): 0}),
+                    ],
+                    any_order=True,
+                )
                 assert consumer_b_on_revoke.mock_calls == []
             else:
                 assert consumer_a_on_assign.mock_calls == [

tests/backends/test_kafka.py

@@ -271,10 +271,13 @@ class TestKafkaStreamsIncrementalRebalancing(TestKafkaStreams):
     cooperative_sticky = True
 
 
-@pytest.mark.skip("kip-848 not functional yet")
 class TestKafkaStreamsKip848(TestKafkaStreams):
     kip_848 = True
 
+    @pytest.mark.xfail(reason="To be fixed")
+    def test_pause_resume_rebalancing(self) -> None:
+        super().test_pause_resume_rebalancing()
+
 
 def test_commit_codec() -> None:
     commit = Commit(