sentry-arroyo 2.20.10__py3-none-any.whl → 2.22.0__py3-none-any.whl

arroyo/backends/abstract.py

@@ -4,16 +4,7 @@ import logging
 import time
 from abc import ABC, abstractmethod, abstractproperty
 from concurrent.futures import Future
-from typing import (
-    Callable,
-    Generic,
-    Mapping,
-    Optional,
-    Protocol,
-    Sequence,
-    TypeVar,
-    Union,
-)
+from typing import Callable, Generic, Mapping, Optional, Sequence, TypeVar, Union
 
 from arroyo.types import BrokerValue, Partition, Topic, TStrategyPayload
 
@@ -187,32 +178,11 @@ class Consumer(Generic[TStrategyPayload], ABC):
         raise NotImplementedError
 
 
-class ProducerFuture(Protocol, Generic[T]):
-    """
-    An abstract interface for a kind of Future. Stdlib futures are too slow to
-    construct, so we use these.
-    """
-
-    def done(self) -> bool:
-        ...
-
-    def result(self, timeout: float | None = None) -> T:
-        """
-        Return result or raise exception. May block, but does not have to.
-        """
-        ...
-
-    def set_result(self, result: T) -> None:
-        ...
-
-    def set_exception(self, exception: Exception) -> None:
-        ...
-
-
 class SimpleProducerFuture(Generic[T]):
     """
     A stub for concurrent.futures.Future that does not construct any Condition
-    variables, therefore is faster to construct.
+    variables, therefore is faster to construct. However, some methods are
+    missing, and result() in particular is not efficient with timeout > 0.
     """
 
     def __init__(self) -> None:
@@ -232,6 +202,10 @@ class SimpleProducerFuture(Generic[T]):
         # only in tests at most. It is only here for the sake of implementing
         # the contract. If you really need result with timeout>0, you should
         # use the stdlib future.
+        #
+        # If this becomes performance sensitive, we can potentially implement
+        # something more sophisticated such as lazily creating the condition
+        # variable, and synchronizing the creation of that using a global lock.
         while deadline is None or time.time() < deadline:
             if self.result_exception is not None:
                 raise self.result_exception
@@ -248,6 +222,9 @@ class SimpleProducerFuture(Generic[T]):
         self.result_exception = exception
 
 
+ProducerFuture = Union[SimpleProducerFuture[T], Future[T]]
+
+
 class Producer(Generic[TStrategyPayload], ABC):
     @abstractmethod
     def produce(

arroyo/processing/strategies/run_task_with_multiprocessing.py

@@ -196,7 +196,7 @@ class BatchBuilder(Generic[TBatchValue]):
 
 
 def parallel_worker_initializer(
-    custom_initialize_func: Optional[Callable[[], None]] = None
+    custom_initialize_func: Optional[Callable[[], None]] = None,
 ) -> None:
     # Worker process should ignore ``SIGINT`` so that processing is not
     # interrupted by ``KeyboardInterrupt`` during graceful shutdown.
@@ -468,6 +468,20 @@ class RunTaskWithMultiprocessing(
     is applying backpressure. You can likely reduce ``num_processes`` and won't
     notice a performance regression.
 
+    Prefetching
+    ~~~~~~~~~~~
+
+    If you set ``prefetch_batches`` to `True`, Arroyo will allocate twice as
+    many input blocks as processes, and will prefetch the next batch while the
+    current batch is being processed. This can help saturate the process pool to
+    increase throughput, but it also increases memory usage.
+
+    Use this option if your consumer is bottlenecked on the multiprocessing step
+    but also runs time-consuming tasks in the other steps, like ``Produce`` or
+    ``Unfold``. By prefetching batches, the pool can immediately start working
+    on the next batch while the current batch is being sent through the next
+    steps.
+
     How to tune your consumer
     ~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -507,6 +521,7 @@ class RunTaskWithMultiprocessing(
         output_block_size: Optional[int] = None,
         max_input_block_size: Optional[int] = None,
         max_output_block_size: Optional[int] = None,
+        prefetch_batches: bool = False,
     ) -> None:
         self.__transform_function = function
         self.__next_step = next_step
@@ -525,18 +540,26 @@ class RunTaskWithMultiprocessing(
         self.__shared_memory_manager = SharedMemoryManager()
         self.__shared_memory_manager.start()
 
+        block_count = num_processes
+        if prefetch_batches:
+            # Allocate twice as many blocks as processes to ensure that every
+            # process can immediately continue to handle another batch while the
+            # main strategy is busy to submit the transformed messages to the
+            # next step.
+            block_count *= 2
+
         self.__input_blocks = [
             self.__shared_memory_manager.SharedMemory(
                 input_block_size or DEFAULT_INPUT_BLOCK_SIZE
             )
-            for _ in range(num_processes)
+            for _ in range(block_count)
         ]
 
         self.__output_blocks = [
             self.__shared_memory_manager.SharedMemory(
                 output_block_size or DEFAULT_OUTPUT_BLOCK_SIZE
             )
-            for _ in range(num_processes)
+            for _ in range(block_count)
         ]
 
         self.__batch_builder: Optional[

{sentry_arroyo-2.20.10.dist-info → sentry_arroyo-2.22.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.20.10
+Version: 2.22.0
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.20.10.dist-info → sentry_arroyo-2.22.0.dist-info}/RECORD

@@ -5,7 +5,7 @@ arroyo/errors.py,sha256=IbtoIbz_m5QrxNRBLOxiy-hOfJQTEwNPCyq6yqedJYk,1059
 arroyo/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/types.py,sha256=sLY0x030np4UmbaW5C1KH1se7Z2pjQiPvAe5x2sXf7A,5684
 arroyo/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arroyo/backends/abstract.py,sha256=qnyuAcbxTIImEXTDA9T1V0Vh4GH_CImnXezK0E3tH5E,9539
+arroyo/backends/abstract.py,sha256=NQ5xG9rjchSUW8URl1WtSYSzMFtwRKB6wtJhWmDyR9E,9386
 arroyo/backends/kafka/__init__.py,sha256=TZ0omd3LFXcZUaN_soFTuTgbjNEQYF1mF_i6_KIfCNo,306
 arroyo/backends/kafka/commit.py,sha256=LPsjvX5PPXR62DT6sa5GuSF78qk9F_L--Fz4kw7-m-s,3060
 arroyo/backends/kafka/configuration.py,sha256=D9zre9H2xagUsk7qBA-bm38V3_4Mg_X5hpKsKx2BkM8,3048
@@ -30,7 +30,7 @@ arroyo/processing/strategies/produce.py,sha256=LUsg2bsVsguc2fTbueTbqLeg3mbnk_FOf
 arroyo/processing/strategies/reduce.py,sha256=xv9bYisgHHyS8fVD1PdGi4TJsaK-4RAhMEDh4WHhYfI,3933
 arroyo/processing/strategies/run_task.py,sha256=MGe2UcIWN7FkPc9plKzRVUNbZ7Sk0jWjw1z2vVOFI_I,2160
 arroyo/processing/strategies/run_task_in_threads.py,sha256=f1sb2AG-BLz11X78jfhtERIkdFogrV8vtdT3pyJdkx0,6144
-arroyo/processing/strategies/run_task_with_multiprocessing.py,sha256=jOL6SKLyrTuCsOkFDl5eOLQzr79bo_yZiuE1d9rhnfA,34558
+arroyo/processing/strategies/run_task_with_multiprocessing.py,sha256=YQ-H3b-9ARnUzeMBzIFSc0TO9c6QIwpZp6A1xlw6JGE,35612
 arroyo/processing/strategies/unfold.py,sha256=bi47pwmKGT0Irsx0HdB7Bhc5hb-yYqLF_xcv3g1ewTk,4231
 arroyo/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/utils/clock.py,sha256=r2EMO4nL5qIb1xnAd1sTAk2yK1UltyUi04lk5BqWKIc,944
@@ -46,11 +46,11 @@ examples/transform_and_produce/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 examples/transform_and_produce/batched.py,sha256=st2R6qTneAtV0JFbKP30Ti3sJDYj8Jkbmta9JckKdZU,2636
 examples/transform_and_produce/script.py,sha256=8kSMIjQNqGYEVyE0PvrfJh-a_UYCrJSstTp_De7kyyg,2306
 examples/transform_and_produce/simple.py,sha256=H7xqxItjl4tx34wVW5dy6mB9G39QucAtxkJSBzVmjgA,1637
-sentry_arroyo-2.20.10.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
+sentry_arroyo-2.22.0.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
 tests/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/backends/mixins.py,sha256=sfNyE0VTeiD3GHOnBYl-9urvPuURI2G1BWke0cz7Dvc,20445
 tests/backends/test_commit.py,sha256=iTHfK1qsBxim0XwxgMvNNSMqDUMEHoYkYBDcgxGBFbs,831
-tests/backends/test_kafka.py,sha256=XGljy72Juesc1qbVs0D02OJitFnry72fRWQdbTqmBp4,12371
+tests/backends/test_kafka.py,sha256=6W6EA41X-ECTfcOeivhQxURnmV2Y0fYy-UzDCnJgDsU,12830
 tests/backends/test_local.py,sha256=Mfd4DFuWVSVtl1GomQ6TIoWuJNcAliKqKU0BShPlEMY,3363
 tests/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/processing/test_processor.py,sha256=2uQcbNeiyuPUlpThq9KTBs_fz5JY-BqZE5Fz4Dxspl0,23274
@@ -72,7 +72,7 @@ tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/utils/test_concurrent.py,sha256=Gwdzym2UZ1HO3rhOSGmzxImWcLFygY8P7MXHT3Q0xTE,455
 tests/utils/test_metrics.py,sha256=bI0EtGgPokMQyEqX58i0-8zvLfxRP2nWaWr2wLMaJ_o,917
 tests/utils/test_retries.py,sha256=AxJLkXWeL9AjHv_p1n0pe8CXXJp24ZQIuYBHfNcmiz4,3075
-sentry_arroyo-2.20.10.dist-info/METADATA,sha256=zxvIwG5Fnbol5xK6YUeGT4_R2YwQgYjHXEudCdO74mA,2201
-sentry_arroyo-2.20.10.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-sentry_arroyo-2.20.10.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
-sentry_arroyo-2.20.10.dist-info/RECORD,,
+sentry_arroyo-2.22.0.dist-info/METADATA,sha256=zDfd2uzTtPpa0GqIxsFpKwZZtSUAEx3GUlKw0mKXBHQ,2200
+sentry_arroyo-2.22.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+sentry_arroyo-2.22.0.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
+sentry_arroyo-2.22.0.dist-info/RECORD,,

tests/backends/test_kafka.py

@@ -167,6 +167,15 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
                 else:
                     raise AssertionError("expected EndOfPartition error")
 
+    @pytest.mark.parametrize("use_simple_futures", [True, False])
+    def test_producer_future_behavior(self, use_simple_futures: bool) -> None:
+        with self.get_topic() as topic:
+            with closing(self.get_producer(use_simple_futures)) as producer:
+                future = producer.produce(topic, next(self.get_payloads()))
+                assert not future.done()
+                assert future.result(5.0)
+                assert future.done()
+
     def test_lenient_offset_reset_latest(self) -> None:
         payload = KafkaPayload(b"a", b"0", [])
         with self.get_topic() as topic: