sentry-arroyo 2.20.8__tar.gz → 2.20.9__tar.gz

{sentry_arroyo-2.20.8/sentry_arroyo.egg-info → sentry_arroyo-2.20.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.20.8
+Version: 2.20.9
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.20.8 → sentry_arroyo-2.20.9}/arroyo/backends/abstract.py

@@ -4,16 +4,7 @@ import logging
 import time
 from abc import ABC, abstractmethod, abstractproperty
 from concurrent.futures import Future
-from typing import (
-    Callable,
-    Generic,
-    Mapping,
-    Optional,
-    Protocol,
-    Sequence,
-    TypeVar,
-    Union,
-)
+from typing import Callable, Generic, Mapping, Optional, Sequence, TypeVar, Union
 
 from arroyo.types import BrokerValue, Partition, Topic, TStrategyPayload
 
@@ -187,32 +178,11 @@ class Consumer(Generic[TStrategyPayload], ABC):
         raise NotImplementedError
 
 
-class ProducerFuture(Protocol, Generic[T]):
-    """
-    An abstract interface for a kind of Future. Stdlib futures are too slow to
-    construct, so we use these.
-    """
-
-    def done(self) -> bool:
-        ...
-
-    def result(self, timeout: float | None = None) -> T:
-        """
-        Return result or raise exception. May block, but does not have to.
-        """
-        ...
-
-    def set_result(self, result: T) -> None:
-        ...
-
-    def set_exception(self, exception: Exception) -> None:
-        ...
-
-
 class SimpleProducerFuture(Generic[T]):
     """
     A stub for concurrent.futures.Future that does not construct any Condition
-    variables, therefore is faster to construct.
+    variables, therefore is faster to construct. However, some methods are
+    missing, and result() in particular is not efficient with timeout > 0.
     """
 
     def __init__(self) -> None:
@@ -232,6 +202,10 @@ class SimpleProducerFuture(Generic[T]):
         # only in tests at most. It is only here for the sake of implementing
         # the contract. If you really need result with timeout>0, you should
         # use the stdlib future.
+        #
+        # If this becomes performance sensitive, we can potentially implement
+        # something more sophisticated such as lazily creating the condition
+        # variable, and synchronizing the creation of that using a global lock.
         while deadline is None or time.time() < deadline:
             if self.result_exception is not None:
                 raise self.result_exception
@@ -248,6 +222,9 @@ class SimpleProducerFuture(Generic[T]):
         self.result_exception = exception
 
 
+ProducerFuture = Union[SimpleProducerFuture[T], Future[T]]
+
+
 class Producer(Generic[TStrategyPayload], ABC):
     @abstractmethod
     def produce(

{sentry_arroyo-2.20.8 → sentry_arroyo-2.20.9}/arroyo/processing/processor.py

@@ -425,9 +425,7 @@ class StreamProcessor(Generic[TStrategyPayload]):
             if self.__message is not None:
                 try:
                     start_submit = time.time()
-                    message = (
-                        Message(self.__message) if self.__message is not None else None
-                    )
+                    message = Message(self.__message)
                     self.__processing_strategy.submit(message)
 
                     self.__metrics_buffer.incr_timing(
@@ -483,6 +481,11 @@ class StreamProcessor(Generic[TStrategyPayload]):
                 except InvalidMessage as e:
                     self._handle_invalid_message(e)
 
+                    if self.__is_paused:
+                        self.__metrics_buffer.incr_counter("arroyo.consumer.resume", 1)
+                        self.__consumer.resume([*self.__consumer.tell().keys()])
+                        self.__is_paused = False
+
                 else:
                     # Resume if we are currently in a paused state
                     if self.__is_paused:

{sentry_arroyo-2.20.8 → sentry_arroyo-2.20.9/sentry_arroyo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.20.8
+Version: 2.20.9
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.20.8 → sentry_arroyo-2.20.9}/setup.py

@@ -10,7 +10,7 @@ def get_requirements() -> Sequence[str]:
 
 setup(
     name="sentry-arroyo",
-    version="2.20.8",
+    version="2.20.9",
     author="Sentry",
     author_email="oss@sentry.io",
     license="Apache-2.0",

{sentry_arroyo-2.20.8 → sentry_arroyo-2.20.9}/tests/backends/test_kafka.py

@@ -167,6 +167,15 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
                 else:
                     raise AssertionError("expected EndOfPartition error")
 
+    @pytest.mark.parametrize("use_simple_futures", [True, False])
+    def test_producer_future_behavior(self, use_simple_futures: bool) -> None:
+        with self.get_topic() as topic:
+            with closing(self.get_producer(use_simple_futures)) as producer:
+                future = producer.produce(topic, next(self.get_payloads()))
+                assert not future.done()
+                assert future.result(5.0)
+                assert future.done()
+
     def test_lenient_offset_reset_latest(self) -> None:
         payload = KafkaPayload(b"a", b"0", [])
         with self.get_topic() as topic:

{sentry_arroyo-2.20.8 → sentry_arroyo-2.20.9}/tests/processing/test_processor.py

@@ -622,3 +622,72 @@ def test_healthcheck(tmpdir: py.path.local) -> None:
 
     processor._run_once()
     assert tmpdir.join("health.txt").mtime() == health_mtime
+
+
+def test_processor_pause_with_invalid_message() -> None:
+
+    topic = Topic("topic")
+
+    consumer = mock.Mock()
+    strategy = mock.Mock()
+    factory = mock.Mock()
+    factory.create_with_partitions.return_value = strategy
+
+    processor: StreamProcessor[int] = StreamProcessor(
+        consumer, topic, factory, IMMEDIATE
+    )
+
+    # Subscribe to topic
+    subscribe_args, subscribe_kwargs = consumer.subscribe.call_args
+    assert subscribe_args[0] == [topic]
+
+    # Partition assignment
+    partition = Partition(topic, 0)
+    consumer.tell.return_value = {}
+    assignment_callback = subscribe_kwargs["on_assign"]
+    offsets = {partition: 0}
+    assignment_callback(offsets)
+
+    # Message that we will get from polling
+    message = Message(BrokerValue(0, partition, 0, datetime.now()))
+
+    # Message will be rejected
+    consumer.poll.return_value = message.value
+    strategy.submit.side_effect = MessageRejected()
+    with assert_changes(lambda: int(consumer.pause.call_count), 0, 1):
+        processor._run_once()
+        assert strategy.submit.call_args_list[-1] == mock.call(message)
+
+        with mock.patch("time.time", return_value=time.time() + 5):
+            processor._run_once()  # Should pause now
+
+    # Consumer is in paused state
+    # The same rejected message should be carried over
+
+    # All partitions are paused
+    consumer.paused.return_value = set(p for p in offsets)
+    # Simulate a continuous backpressure state where messages are being rejected
+    strategy.submit.side_effect = MessageRejected()
+
+    # Simulate Kafka returning nothing since the consumer is paused
+    consumer.poll.return_value = None
+
+    # The next poll returns nothing, but we are still carrying over the rejected message
+    processor._run_once()
+    assert consumer.poll.return_value is None
+
+    # At this point, let's say the message carried over is invalid (e.g. it could be stale)
+    strategy.submit.side_effect = InvalidMessage(partition, 0, needs_commit=False)
+
+    # Handles the invalid message and unpauses the consumer
+    with assert_changes(lambda: int(consumer.resume.call_count), 0, 1):
+        processor._run_once()
+
+    # Poll for the next message from Kafka
+    new_message = Message(BrokerValue(0, partition, 1, datetime.now()))
+    consumer.poll.return_value = new_message.value
+    strategy.submit.return_value = None
+    strategy.submit.side_effect = None
+
+    processor._run_once()
+    assert strategy.submit.call_args_list[-1] == mock.call(new_message)