sentry-arroyo 2.21.0__tar.gz → 2.23.0__tar.gz

{sentry_arroyo-2.21.0/sentry_arroyo.egg-info → sentry_arroyo-2.23.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.21.0
+Version: 2.23.0
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: confluent-kafka>=2.7.0
+Requires-Dist: confluent-kafka<2.10.0,>=2.7.0
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{sentry_arroyo-2.21.0 → sentry_arroyo-2.23.0}/arroyo/processing/processor.py

@@ -236,7 +236,7 @@ class StreamProcessor(Generic[TStrategyPayload]):
             logger.info("New partitions assigned: %r", partitions)
             logger.info("Member id: %r", self.__consumer.member_id)
             self.__metrics_buffer.metrics.increment(
-                "arroyo.consumer.partitions_assigned.count", len(partitions)
+                "arroyo.consumer.partitions_assigned.count", len(partitions), tags={"consumer_member_id": self.__consumer.member_id}
             )
 
             current_partitions = dict(self.__consumer.tell())
@@ -262,7 +262,7 @@ class StreamProcessor(Generic[TStrategyPayload]):
             logger.info("Partitions to revoke: %r", partitions)
 
             self.__metrics_buffer.metrics.increment(
-                "arroyo.consumer.partitions_revoked.count", len(partitions)
+                "arroyo.consumer.partitions_revoked.count", len(partitions), tags={"consumer_member_id": self.__consumer.member_id}
             )
 
             if partitions:

{sentry_arroyo-2.21.0 → sentry_arroyo-2.23.0}/arroyo/processing/strategies/run_task_with_multiprocessing.py

@@ -196,7 +196,7 @@ class BatchBuilder(Generic[TBatchValue]):
 
 
 def parallel_worker_initializer(
-    custom_initialize_func: Optional[Callable[[], None]] = None
+    custom_initialize_func: Optional[Callable[[], None]] = None,
 ) -> None:
     # Worker process should ignore ``SIGINT`` so that processing is not
     # interrupted by ``KeyboardInterrupt`` during graceful shutdown.
@@ -468,6 +468,20 @@ class RunTaskWithMultiprocessing(
     is applying backpressure. You can likely reduce ``num_processes`` and won't
     notice a performance regression.
 
+    Prefetching
+    ~~~~~~~~~~~
+
+    If you set ``prefetch_batches`` to `True`, Arroyo will allocate twice as
+    many input blocks as processes, and will prefetch the next batch while the
+    current batch is being processed. This can help saturate the process pool to
+    increase throughput, but it also increases memory usage.
+
+    Use this option if your consumer is bottlenecked on the multiprocessing step
+    but also runs time-consuming tasks in the other steps, like ``Produce`` or
+    ``Unfold``. By prefetching batches, the pool can immediately start working
+    on the next batch while the current batch is being sent through the next
+    steps.
+
     How to tune your consumer
     ~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -507,6 +521,7 @@ class RunTaskWithMultiprocessing(
         output_block_size: Optional[int] = None,
         max_input_block_size: Optional[int] = None,
         max_output_block_size: Optional[int] = None,
+        prefetch_batches: bool = False,
     ) -> None:
         self.__transform_function = function
         self.__next_step = next_step
@@ -525,18 +540,26 @@ class RunTaskWithMultiprocessing(
         self.__shared_memory_manager = SharedMemoryManager()
         self.__shared_memory_manager.start()
 
+        block_count = num_processes
+        if prefetch_batches:
+            # Allocate twice as many blocks as processes to ensure that every
+            # process can immediately continue to handle another batch while the
+            # main strategy is busy to submit the transformed messages to the
+            # next step.
+            block_count *= 2
+
         self.__input_blocks = [
             self.__shared_memory_manager.SharedMemory(
                 input_block_size or DEFAULT_INPUT_BLOCK_SIZE
             )
-            for _ in range(num_processes)
+            for _ in range(block_count)
         ]
 
         self.__output_blocks = [
             self.__shared_memory_manager.SharedMemory(
                 output_block_size or DEFAULT_OUTPUT_BLOCK_SIZE
             )
-            for _ in range(num_processes)
+            for _ in range(block_count)
         ]
 
         self.__batch_builder: Optional[

sentry_arroyo-2.23.0/requirements.txt

@@ -0,0 +1,3 @@
+# It seems 2.10.0 is problematic for the kip 848 tests.
+# It needs some time to be fixed.
+confluent-kafka>=2.7.0,<2.10.0

{sentry_arroyo-2.21.0 → sentry_arroyo-2.23.0/sentry_arroyo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.21.0
+Version: 2.23.0
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: confluent-kafka>=2.7.0
+Requires-Dist: confluent-kafka<2.10.0,>=2.7.0
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

sentry_arroyo-2.23.0/sentry_arroyo.egg-info/requires.txt

@@ -0,0 +1 @@
+confluent-kafka<2.10.0,>=2.7.0

{sentry_arroyo-2.21.0 → sentry_arroyo-2.23.0}/setup.py

@@ -10,7 +10,7 @@ def get_requirements() -> Sequence[str]:
 
 setup(
     name="sentry-arroyo",
-    version="2.21.0",
+    version="2.23.0",
     author="Sentry",
     author_email="oss@sentry.io",
     license="Apache-2.0",

sentry_arroyo-2.21.0/requirements.txt

@@ -1 +0,0 @@
-confluent-kafka>=2.7.0

sentry_arroyo-2.21.0/sentry_arroyo.egg-info/requires.txt

@@ -1 +0,0 @@
-confluent-kafka>=2.7.0