sentry-arroyo 2.19.5__tar.gz → 2.19.7__tar.gz

{sentry_arroyo-2.19.5/sentry_arroyo.egg-info → sentry_arroyo-2.19.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sentry-arroyo
-Version: 2.19.5
+Version: 2.19.7
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7}/arroyo/backends/kafka/commit.py

@@ -97,6 +97,6 @@ class CommitCodec(Codec[KafkaPayload, Commit]):
 
         if max_times_to_log_legacy_message > 0:
             max_times_to_log_legacy_message -= 1
-            logger.warn(f"Legacy commit message found: {commit}")
+            logger.warning("Legacy commit message found: %s", commit)
 
         return commit

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7}/arroyo/backends/kafka/consumer.py

@@ -161,6 +161,10 @@ class KafkaConsumer(Consumer[KafkaPayload]):
         )
 
         configuration = dict(configuration)
+        self.__is_incremental = (
+            configuration.get("partition.assignment.strategy") == "cooperative-sticky"
+            or configuration.get("group.protocol") == "consumer"
+        )
         auto_offset_reset = configuration.get("auto.offset.reset", "largest")
 
         # This is a special flag that controls the auto offset behavior for
@@ -269,6 +273,10 @@ class KafkaConsumer(Consumer[KafkaPayload]):
         def assignment_callback(
             consumer: ConfluentConsumer, partitions: Sequence[ConfluentTopicPartition]
         ) -> None:
+            if not partitions:
+                logger.info("skipping empty assignment")
+                return
+
             self.__state = KafkaConsumerState.ASSIGNING
 
             try:
@@ -451,12 +459,14 @@ class KafkaConsumer(Consumer[KafkaPayload]):
 
     def __assign(self, offsets: Mapping[Partition, int]) -> None:
         self.__validate_offsets(offsets)
-        self.__consumer.assign(
-            [
-                ConfluentTopicPartition(partition.topic.name, partition.index, offset)
-                for partition, offset in offsets.items()
-            ]
-        )
+        partitions = [
+            ConfluentTopicPartition(partition.topic.name, partition.index, offset)
+            for partition, offset in offsets.items()
+        ]
+        if self.__is_incremental:
+            self.__consumer.incremental_assign(partitions)
+        else:
+            self.__consumer.assign(partitions)
         self.__offsets.update(offsets)
 
     def seek(self, offsets: Mapping[Partition, int]) -> None:

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7}/arroyo/dlq.py

@@ -44,11 +44,12 @@ class InvalidMessage(Exception):
     """
 
     def __init__(
-        self, partition: Partition, offset: int, needs_commit: bool = True
+        self, partition: Partition, offset: int, needs_commit: bool = True, reason: Optional[str] = None,
     ) -> None:
         self.partition = partition
         self.offset = offset
         self.needs_commit = needs_commit
+        self.reason = reason
 
     @classmethod
     def from_value(cls, value: BrokerValue[Any]) -> InvalidMessage:
@@ -177,7 +178,7 @@ class DlqLimitState:
 class DlqProducer(ABC, Generic[TStrategyPayload]):
     @abstractmethod
     def produce(
-        self, value: BrokerValue[TStrategyPayload]
+        self, value: BrokerValue[TStrategyPayload], reason: Optional[str] = None
     ) -> Future[BrokerValue[TStrategyPayload]]:
         """
         Produce a message to DLQ.
@@ -201,7 +202,7 @@ class NoopDlqProducer(DlqProducer[Any]):
     """
 
     def produce(
-        self, value: BrokerValue[KafkaPayload]
+        self, value: BrokerValue[KafkaPayload], reason: Optional[str] = None,
     ) -> Future[BrokerValue[KafkaPayload]]:
         future: Future[BrokerValue[KafkaPayload]] = Future()
         future.set_running_or_notify_cancel()
@@ -229,7 +230,9 @@ class KafkaDlqProducer(DlqProducer[KafkaPayload]):
         self.__topic = topic
 
     def produce(
-        self, value: BrokerValue[KafkaPayload]
+        self,
+        value: BrokerValue[KafkaPayload],
+        reason: Optional[str] = None,
     ) -> Future[BrokerValue[KafkaPayload]]:
         value.payload.headers.append(
             ("original_partition", f"{value.partition.index}".encode("utf-8"))
@@ -315,11 +318,11 @@ class BufferedMessages(Generic[TStrategyPayload]):
 
         return None
 
-    def reset(self) -> None:
+    def remove(self, partition: Partition) -> None:
         """
-        Reset the buffer.
+        Remove a revoked partition from the buffer.
         """
-        self.__buffered_messages = defaultdict(deque)
+        self.__buffered_messages.pop(partition, None)
 
 
 class DlqPolicyWrapper(Generic[TStrategyPayload]):
@@ -343,9 +346,9 @@ class DlqPolicyWrapper(Generic[TStrategyPayload]):
                 ]
             ],
         ] = defaultdict(deque)
-        self.reset_offsets({})
+        self.reset_dlq_limits({})
 
-    def reset_offsets(self, assignment: Mapping[Partition, int]) -> None:
+    def reset_dlq_limits(self, assignment: Mapping[Partition, int]) -> None:
         """
         Called on consumer assignment
         """
@@ -353,7 +356,7 @@ class DlqPolicyWrapper(Generic[TStrategyPayload]):
             self.__dlq_policy.limit, assignment
         )
 
-    def produce(self, message: BrokerValue[TStrategyPayload]) -> None:
+    def produce(self, message: BrokerValue[TStrategyPayload], reason: Optional[str] = None) -> None:
         """
         Removes all completed futures, then appends the given future to the list.
         Blocks if the list is full. If the DLQ limit is exceeded, an exception is raised.
@@ -371,7 +374,7 @@ class DlqPolicyWrapper(Generic[TStrategyPayload]):
 
         should_accept = self.__dlq_limit_state.record_invalid_message(message)
         if should_accept:
-            future = self.__dlq_policy.producer.produce(message)
+            future = self.__dlq_policy.producer.produce(message, reason)
             self.__futures[message.partition].append((message, future))
         else:
             raise RuntimeError("Dlq limit exceeded")

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7}/arroyo/processing/processor.py

@@ -238,16 +238,23 @@ class StreamProcessor(Generic[TStrategyPayload]):
                 "arroyo.consumer.partitions_assigned.count", len(partitions)
             )
 
-            self.__buffered_messages.reset()
+            current_partitions = dict(self.__consumer.tell())
+            current_partitions.update(partitions)
+
             if self.__dlq_policy:
-                self.__dlq_policy.reset_offsets(partitions)
-            if partitions:
+                self.__dlq_policy.reset_dlq_limits(current_partitions)
+            if current_partitions:
                 if self.__processing_strategy is not None:
-                    logger.exception(
+                    # TODO: for cooperative-sticky rebalancing this can happen
+                    # quite often. we should port the changes to
+                    # ProcessingStrategyFactory that we made in Rust: Remove
+                    # create_with_partitions, replace with create +
+                    # update_partitions
+                    logger.warning(
                         "Partition assignment while processing strategy active"
                     )
                     _close_strategy()
-                _create_strategy(partitions)
+                _create_strategy(current_partitions)
 
         @_rdkafka_callback(metrics=self.__metrics_buffer)
         def on_partitions_revoked(partitions: Sequence[Partition]) -> None:
@@ -278,6 +285,9 @@ class StreamProcessor(Generic[TStrategyPayload]):
                 except RuntimeError:
                     pass
 
+            for partition in partitions:
+                self.__buffered_messages.remove(partition)
+
             # Partition revocation can happen anytime during the consumer lifecycle and happen
             # multiple times. What we want to know is that the consumer is not stuck somewhere.
             # The presence of this message as the last message of a consumer
@@ -366,7 +376,7 @@ class StreamProcessor(Generic[TStrategyPayload]):
                 ) from None
 
             # XXX: This blocks if there are more than MAX_PENDING_FUTURES in the queue.
-            self.__dlq_policy.produce(invalid_message)
+            self.__dlq_policy.produce(invalid_message, exc.reason)
 
             self.__metrics_buffer.incr_timing(
                 "arroyo.consumer.dlq.time", time.time() - start_dlq

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7}/arroyo/utils/clock.py

@@ -28,7 +28,7 @@ class SystemClock(Clock):
         time.sleep(duration)
 
 
-class TestingClock(Clock):
+class MockedClock(Clock):
     """
     A clock implementation that uses a stable time for testing. To advance
     the time, use the ``sleep`` method.

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7/sentry_arroyo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sentry-arroyo
-Version: 2.19.5
+Version: 2.19.7
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7}/setup.py

@@ -10,7 +10,7 @@ def get_requirements() -> Sequence[str]:
 
 setup(
     name="sentry-arroyo",
-    version="2.19.5",
+    version="2.19.7",
     author="Sentry",
     author_email="oss@sentry.io",
     license="Apache-2.0",

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7}/tests/backends/mixins.py

@@ -2,7 +2,7 @@ import time
 import uuid
 from abc import ABC, abstractmethod
 from contextlib import closing
-from typing import ContextManager, Generic, Iterator, Mapping, Optional, Sequence
+from typing import Any, ContextManager, Generic, Iterator, Mapping, Optional, Sequence
 from unittest import mock
 
 import pytest
@@ -14,6 +14,8 @@ from tests.assertions import assert_changes, assert_does_not_change
 
 
 class StreamsTestMixin(ABC, Generic[TStrategyPayload]):
+    cooperative_sticky = False
+
     @abstractmethod
     def get_topic(self, partitions: int = 1) -> ContextManager[Topic]:
         raise NotImplementedError
@@ -397,6 +399,11 @@ class StreamsTestMixin(ABC, Generic[TStrategyPayload]):
     def test_pause_resume_rebalancing(self) -> None:
         payloads = self.get_payloads()
 
+        consumer_a_on_assign = mock.Mock()
+        consumer_a_on_revoke = mock.Mock()
+        consumer_b_on_assign = mock.Mock()
+        consumer_b_on_revoke = mock.Mock()
+
         with self.get_topic(2) as topic, closing(
             self.get_producer()
         ) as producer, closing(
@@ -408,10 +415,22 @@ class StreamsTestMixin(ABC, Generic[TStrategyPayload]):
                 producer.produce(Partition(topic, i), next(payloads)).result(
                     timeout=5.0
                 )
-                for i in range(2)
+                for i in [0, 1]
             ]
 
-            consumer_a.subscribe([topic])
+            def wait_until_rebalancing(
+                from_consumer: Consumer[Any], to_consumer: Consumer[Any]
+            ) -> None:
+                for _ in range(10):
+                    assert from_consumer.poll(0) is None
+                    if to_consumer.poll(1.0) is not None:
+                        return
+
+                raise RuntimeError("no rebalancing happened")
+
+            consumer_a.subscribe(
+                [topic], on_assign=consumer_a_on_assign, on_revoke=consumer_a_on_revoke
+            )
 
             # It doesn't really matter which message is fetched first -- we
             # just want to know the assignment occurred.
@@ -428,19 +447,69 @@ class StreamsTestMixin(ABC, Generic[TStrategyPayload]):
                 [Partition(topic, 0), Partition(topic, 1)]
             )
 
-            consumer_b.subscribe([topic])
-            for i in range(10):
-                assert consumer_a.poll(0) is None  # attempt to force session timeout
-                if consumer_b.poll(1.0) is not None:
-                    break
-            else:
-                assert False, "rebalance did not occur"
+            consumer_b.subscribe(
+                [topic], on_assign=consumer_b_on_assign, on_revoke=consumer_b_on_revoke
+            )
+
+            wait_until_rebalancing(consumer_a, consumer_b)
 
-            # The first consumer should have had its offsets rolled back, as
-            # well as should have had it's partition resumed during
-            # rebalancing.
-            assert consumer_a.paused() == []
-            assert consumer_a.poll(10.0) is not None
+            if self.cooperative_sticky:
+                # within incremental rebalancing, only one partition should have been reassigned to the consumer_b, and consumer_a should remain paused
+                assert consumer_a.paused() == [Partition(topic, 1)]
+                assert consumer_a.poll(10.0) is None
+            else:
+                # The first consumer should have had its offsets rolled back, as
+                # well as should have had it's partition resumed during
+                # rebalancing.
+                assert consumer_a.paused() == []
+                assert consumer_a.poll(10.0) is not None
 
             assert len(consumer_a.tell()) == 1
             assert len(consumer_b.tell()) == 1
+
+            (consumer_a_partition,) = consumer_a.tell()
+            (consumer_b_partition,) = consumer_b.tell()
+
+            # Pause consumer_a again.
+            consumer_a.pause(list(consumer_a.tell()))
+            # if we close consumer_a, consumer_b should get all partitions
+            producer.produce(next(iter(consumer_a.tell())), next(payloads)).result(
+                timeout=5.0
+            )
+            consumer_a.unsubscribe()
+            wait_until_rebalancing(consumer_a, consumer_b)
+
+            assert len(consumer_b.tell()) == 2
+
+            if self.cooperative_sticky:
+
+                assert consumer_a_on_assign.mock_calls == [
+                    mock.call({Partition(topic, 0): 0, Partition(topic, 1): 0}),
+                ]
+                assert consumer_a_on_revoke.mock_calls == [
+                    mock.call([Partition(topic, 0)]),
+                    mock.call([Partition(topic, 1)]),
+                ]
+
+                assert consumer_b_on_assign.mock_calls == [
+                    mock.call({Partition(topic, 0): 0}),
+                    mock.call({Partition(topic, 1): 0}),
+                ]
+                assert consumer_b_on_revoke.mock_calls == []
+            else:
+                assert consumer_a_on_assign.mock_calls == [
+                    mock.call({Partition(topic, 0): 0, Partition(topic, 1): 0}),
+                    mock.call({consumer_a_partition: 0}),
+                ]
+                assert consumer_a_on_revoke.mock_calls == [
+                    mock.call([Partition(topic, 0), Partition(topic, 1)]),
+                    mock.call([consumer_a_partition]),
+                ]
+
+                assert consumer_b_on_assign.mock_calls == [
+                    mock.call({consumer_b_partition: 0}),
+                    mock.call({Partition(topic, 0): 0, Partition(topic, 1): 0}),
+                ]
+                assert consumer_b_on_revoke.mock_calls == [
+                    mock.call([consumer_b_partition])
+                ]

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7}/tests/backends/test_kafka.py

@@ -14,7 +14,10 @@ from confluent_kafka.admin import AdminClient, NewTopic
 
 from arroyo.backends.kafka import KafkaConsumer, KafkaPayload, KafkaProducer
 from arroyo.backends.kafka.commit import CommitCodec
-from arroyo.backends.kafka.configuration import build_kafka_configuration
+from arroyo.backends.kafka.configuration import (
+    KafkaBrokerConfig,
+    build_kafka_configuration,
+)
 from arroyo.backends.kafka.consumer import as_kafka_configuration_bool
 from arroyo.commit import IMMEDIATE, Commit
 from arroyo.errors import ConsumerError, EndOfPartition
@@ -56,6 +59,7 @@ def get_topic(
     configuration: Mapping[str, Any], partitions_count: int
 ) -> Iterator[Topic]:
     name = f"test-{uuid.uuid1().hex}"
+    configuration = dict(configuration)
     client = AdminClient(configuration)
     [[key, future]] = client.create_topics(
         [NewTopic(name, num_partitions=partitions_count, replication_factor=1)]
@@ -71,10 +75,15 @@ def get_topic(
 
 
 class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
+    kip_848 = False
 
-    configuration = build_kafka_configuration(
-        {"bootstrap.servers": os.environ.get("DEFAULT_BROKERS", "localhost:9092")}
-    )
+    @property
+    def configuration(self) -> KafkaBrokerConfig:
+        config = {
+            "bootstrap.servers": os.environ.get("DEFAULT_BROKERS", "localhost:9092"),
+        }
+
+        return build_kafka_configuration(config)
 
     @contextlib.contextmanager
     def get_topic(self, partitions: int = 1) -> Iterator[Topic]:
@@ -90,7 +99,7 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
         enable_end_of_partition: bool = True,
         auto_offset_reset: str = "earliest",
         strict_offset_reset: Optional[bool] = None,
-        max_poll_interval_ms: Optional[int] = None
+        max_poll_interval_ms: Optional[int] = None,
     ) -> KafkaConsumer:
         configuration = {
             **self.configuration,
@@ -110,6 +119,16 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
             if max_poll_interval_ms < 45000:
                 configuration["session.timeout.ms"] = max_poll_interval_ms
 
+        if self.cooperative_sticky:
+            configuration["partition.assignment.strategy"] = "cooperative-sticky"
+
+        if self.kip_848:
+            configuration["group.protocol"] = "consumer"
+            configuration.pop("session.timeout.ms")
+            configuration.pop("max.poll.interval.ms", None)
+            assert "group.protocol.type" not in configuration
+            assert "heartbeat.interval.ms" not in configuration
+
         return KafkaConsumer(configuration)
 
     def get_producer(self) -> KafkaProducer:
@@ -210,7 +229,9 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
         poll_interval = 6000
 
         with self.get_topic() as topic:
-            with closing(self.get_producer()) as producer, closing(self.get_consumer(max_poll_interval_ms=poll_interval)) as consumer:
+            with closing(self.get_producer()) as producer, closing(
+                self.get_consumer(max_poll_interval_ms=poll_interval)
+            ) as consumer:
                 producer.produce(topic, next(self.get_payloads())).result(5.0)
 
                 processor = StreamProcessor(consumer, topic, factory, IMMEDIATE)
@@ -245,6 +266,16 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
                 assert consumer.paused() == []
 
 
+class TestKafkaStreamsIncrementalRebalancing(TestKafkaStreams):
+    # re-test the kafka consumer with cooperative-sticky rebalancing
+    cooperative_sticky = True
+
+
+@pytest.mark.skip("kip-848 not functional yet")
+class TestKafkaStreamsKip848(TestKafkaStreams):
+    kip_848 = True
+
+
 def test_commit_codec() -> None:
     commit = Commit(
         "group", Partition(Topic("topic"), 0), 0, time.time(), time.time() - 5

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7}/tests/backends/test_local.py

@@ -18,14 +18,14 @@ from arroyo.backends.local.storages.abstract import (
 )
 from arroyo.backends.local.storages.memory import MemoryMessageStorage
 from arroyo.types import Partition, Topic
-from arroyo.utils.clock import TestingClock
+from arroyo.utils.clock import MockedClock
 from tests.backends.mixins import StreamsTestMixin
 
 
 class LocalStreamsTestMixin(StreamsTestMixin[int]):
     def setUp(self) -> None:
         self.storage = self.get_message_storage()
-        self.broker = LocalBroker(self.storage, TestingClock())
+        self.broker = LocalBroker(self.storage, MockedClock())
 
     @abstractmethod
     def get_message_storage(self) -> MessageStorage[int]:

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7}/tests/processing/strategies/test_produce.py

@@ -9,13 +9,13 @@ from arroyo.backends.local.storages.memory import MemoryMessageStorage
 from arroyo.processing.strategies.abstract import MessageRejected
 from arroyo.processing.strategies.produce import Produce
 from arroyo.types import Message, Partition, Topic, Value
-from arroyo.utils.clock import TestingClock
+from arroyo.utils.clock import MockedClock
 
 
 def test_produce() -> None:
     orig_topic = Topic("orig-topic")
     result_topic = Topic("result-topic")
-    clock = TestingClock()
+    clock = MockedClock()
     broker_storage: MemoryMessageStorage[KafkaPayload] = MemoryMessageStorage()
     broker: LocalBroker[KafkaPayload] = LocalBroker(broker_storage, clock)
     broker.create_topic(result_topic, partitions=1)

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7}/tests/processing/test_processor.py

@@ -19,7 +19,7 @@ from arroyo.processing.strategies.abstract import (
     ProcessingStrategyFactory,
 )
 from arroyo.types import BrokerValue, Commit, Message, Partition, Topic
-from arroyo.utils.clock import TestingClock
+from arroyo.utils.clock import MockedClock
 from tests.assertions import assert_changes, assert_does_not_change
 from tests.metrics import Increment, TestingMetricsBackend, Timing
 
@@ -41,6 +41,7 @@ def test_stream_processor_lifecycle() -> None:
 
     # The processor should accept heartbeat messages without an assignment or
     # active processor.
+    consumer.tell.return_value = {}
     consumer.poll.return_value = None
     processor._run_once()
 
@@ -166,6 +167,7 @@ def test_stream_processor_termination_on_error() -> None:
     offset = 0
     now = datetime.now()
 
+    consumer.tell.return_value = {}
     consumer.poll.return_value = BrokerValue(0, partition, offset, now)
 
     exception = NotImplementedError("error")
@@ -199,6 +201,7 @@ def test_stream_processor_invalid_message_from_poll() -> None:
     offset = 1
     now = datetime.now()
 
+    consumer.tell.return_value = {}
     consumer.poll.side_effect = [BrokerValue(0, partition, offset, now)]
 
     strategy = mock.Mock()
@@ -236,6 +239,7 @@ def test_stream_processor_invalid_message_from_submit() -> None:
     offset = 1
     now = datetime.now()
 
+    consumer.tell.return_value = {}
     consumer.poll.side_effect = [
         BrokerValue(0, partition, offset, now),
         BrokerValue(1, partition, offset + 1, now),
@@ -283,6 +287,7 @@ def test_stream_processor_create_with_partitions() -> None:
     topic = Topic("topic")
 
     consumer = mock.Mock()
+    consumer.tell.return_value = {}
     strategy = mock.Mock()
     factory = mock.Mock()
     factory.create_with_partitions.return_value = strategy
@@ -306,13 +311,15 @@ def test_stream_processor_create_with_partitions() -> None:
     assert factory.create_with_partitions.call_count == 1
     assert create_args[1] == offsets_p0
 
+    consumer.tell.return_value = {**offsets_p0}
+
     # Second partition assigned
     offsets_p1 = {Partition(topic, 1): 0}
     assignment_callback(offsets_p1)
 
     create_args, _ = factory.create_with_partitions.call_args
     assert factory.create_with_partitions.call_count == 2
-    assert create_args[1] == offsets_p1
+    assert create_args[1] == {**offsets_p1, **offsets_p0}
 
     processor._run_once()
 
@@ -376,6 +383,7 @@ def run_commit_policy_test(
 ) -> Sequence[int]:
     commit = mock.Mock()
     consumer = mock.Mock()
+    consumer.tell.return_value = {}
     consumer.commit_offsets = commit
 
     factory = CommitOffsetsFactory()
@@ -523,7 +531,7 @@ def test_commit_policy_bench(
     storage: MessageStorage[int] = MemoryMessageStorage()
     storage.create_topic(topic, num_partitions)
 
-    broker = LocalBroker(storage, TestingClock())
+    broker = LocalBroker(storage, MockedClock())
 
     consumer = broker.get_consumer("test-group", enable_end_of_partition=True)
 
@@ -551,6 +559,7 @@ def test_dlq() -> None:
     partition = Partition(topic, 0)
     consumer = mock.Mock()
     consumer.poll.return_value = BrokerValue(0, partition, 1, datetime.now())
+    consumer.tell.return_value = {}
     strategy = mock.Mock()
     strategy.submit.side_effect = InvalidMessage(partition, 1)
     factory = mock.Mock()
@@ -585,6 +594,7 @@ def test_healthcheck(tmpdir: py.path.local) -> None:
     consumer = mock.Mock()
     now = datetime.now()
     consumer.poll.return_value = BrokerValue(0, partition, 1, now)
+    consumer.tell.return_value = {}
     strategy = mock.Mock()
     strategy.submit.side_effect = InvalidMessage(partition, 1)
     factory = mock.Mock()

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7}/tests/test_dlq.py

@@ -107,7 +107,7 @@ def test_dlq_policy_wrapper() -> None:
     )
     partition = Partition(topic, 0)
     wrapper = DlqPolicyWrapper(dlq_policy)
-    wrapper.reset_offsets({partition: 0})
+    wrapper.reset_dlq_limits({partition: 0})
     wrapper.MAX_PENDING_FUTURES = 1
     for i in range(10):
         message = BrokerValue(KafkaPayload(None, b"", []), partition, i, datetime.now())
@@ -123,7 +123,7 @@ def test_dlq_policy_wrapper_limit_exceeded() -> None:
     )
     partition = Partition(topic, 0)
     wrapper = DlqPolicyWrapper(dlq_policy)
-    wrapper.reset_offsets({partition: 0})
+    wrapper.reset_dlq_limits({partition: 0})
     wrapper.MAX_PENDING_FUTURES = 1
 
     message = BrokerValue(KafkaPayload(None, b"", []), partition, 1, datetime.now())

{sentry_arroyo-2.19.5 → sentry_arroyo-2.19.7}/tests/utils/test_retries.py

@@ -3,7 +3,7 @@ from unittest import mock
 
 import pytest
 
-from arroyo.utils.clock import TestingClock
+from arroyo.utils.clock import MockedClock
 from arroyo.utils.retries import BasicRetryPolicy, RetryException, constant_delay
 
 value = object()
@@ -45,7 +45,7 @@ def setup_function() -> None:
 
 def test_basic_retry_policy_no_delay() -> None:
 
-    clock = TestingClock()
+    clock = MockedClock()
 
     policy = BasicRetryPolicy(3, clock=clock)
 
@@ -68,19 +68,19 @@ def test_basic_retry_policy_no_delay() -> None:
 
 @pytest.mark.parametrize("delay", [1, constant_delay(1)])
 def test_basic_retry_policy_with_delay(delay: int) -> None:
-    clock = TestingClock()
+    clock = MockedClock()
     policy = BasicRetryPolicy(3, delay, clock=clock)
     assert policy.call(good_function) is value
     assert good_function.call_count == 1
     assert clock.time() == 0
 
-    clock = TestingClock()
+    clock = MockedClock()
     policy = BasicRetryPolicy(3, delay, clock=clock)
     assert policy.call(flaky_function) is value
     assert flaky_function.call_count == 2
     assert clock.time() == 1  # one retry
 
-    clock = TestingClock()
+    clock = MockedClock()
     policy = BasicRetryPolicy(3, delay, clock=clock)
     try:
         policy.call(bad_function)
@@ -109,7 +109,7 @@ def test_basic_retry_policy_with_supression() -> None:
     def suppression_test(exception: Exception) -> bool:
         return isinstance(exception, ExpectedError)
 
-    clock = TestingClock()
+    clock = MockedClock()
     policy = BasicRetryPolicy(
         3, constant_delay(1), suppression_test=suppression_test, clock=clock
     )