sentry-arroyo 2.35.0__py3-none-any.whl → 2.37.0__py3-none-any.whl

arroyo/processing/strategies/run_task_with_multiprocessing.py

@@ -42,6 +42,7 @@ DEFAULT_OUTPUT_BLOCK_SIZE = 16 * 1024 * 1024
 
 LOG_THRESHOLD_TIME = 20  # In seconds
 FAIL_THRESHOLD_TIME = 180  # In seconds
+BATCH_PROCESS_LOG_TIME = 5  # In seconds
 
 
 class ChildProcessTerminated(RuntimeError):
@@ -233,6 +234,7 @@ def parallel_run_task_worker_apply(
     output_block: SharedMemory,
     start_index: int = 0,
 ) -> ParallelRunTaskResult[TResult]:
+    last_logged_time = time.time()
     valid_messages_transformed: MessageBatch[
         Union[InvalidMessage, Message[Union[FilteredPayload, TResult]]]
     ] = MessageBatch(
@@ -242,6 +244,14 @@ def parallel_run_task_worker_apply(
 
     next_index_to_process = start_index
     while next_index_to_process < len(input_batch):
+        if last_logged_time + BATCH_PROCESS_LOG_TIME < time.time():
+            logger.info(
+                "Batch process is taking longer than %d seconds. index: %d",
+                BATCH_PROCESS_LOG_TIME,
+                next_index_to_process,
+            )
+            last_logged_time = time.time()
+
         message = input_batch[next_index_to_process]
 
         # Theory: Doing this check in the subprocess is cheaper because we

{sentry_arroyo-2.35.0.dist-info → sentry_arroyo-2.37.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.35.0
+Version: 2.37.0
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.35.0.dist-info → sentry_arroyo-2.37.0.dist-info}/RECORD

@@ -30,7 +30,7 @@ arroyo/processing/strategies/produce.py,sha256=w4GI7KC-CGn2bLG_qPcuKJo0EbZ4PF2TJ
 arroyo/processing/strategies/reduce.py,sha256=xv9bYisgHHyS8fVD1PdGi4TJsaK-4RAhMEDh4WHhYfI,3933
 arroyo/processing/strategies/run_task.py,sha256=MGe2UcIWN7FkPc9plKzRVUNbZ7Sk0jWjw1z2vVOFI_I,2160
 arroyo/processing/strategies/run_task_in_threads.py,sha256=f1sb2AG-BLz11X78jfhtERIkdFogrV8vtdT3pyJdkx0,6144
-arroyo/processing/strategies/run_task_with_multiprocessing.py,sha256=IH21qpXKxN0DN4XBsG26piw84ZJaisx7rBup7nci8m8,37272
+arroyo/processing/strategies/run_task_with_multiprocessing.py,sha256=UpBpYRthCl93_Z-KeEycTbdIXUAor2-R7C_2hqBJmCg,37655
 arroyo/processing/strategies/unfold.py,sha256=bi47pwmKGT0Irsx0HdB7Bhc5hb-yYqLF_xcv3g1ewTk,4231
 arroyo/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/utils/clock.py,sha256=r2EMO4nL5qIb1xnAd1sTAk2yK1UltyUi04lk5BqWKIc,944
@@ -46,7 +46,7 @@ examples/transform_and_produce/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 examples/transform_and_produce/batched.py,sha256=st2R6qTneAtV0JFbKP30Ti3sJDYj8Jkbmta9JckKdZU,2636
 examples/transform_and_produce/script.py,sha256=8kSMIjQNqGYEVyE0PvrfJh-a_UYCrJSstTp_De7kyyg,2306
 examples/transform_and_produce/simple.py,sha256=H7xqxItjl4tx34wVW5dy6mB9G39QucAtxkJSBzVmjgA,1637
-sentry_arroyo-2.35.0.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
+sentry_arroyo-2.37.0.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
 tests/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/backends/mixins.py,sha256=sfNyE0VTeiD3GHOnBYl-9urvPuURI2G1BWke0cz7Dvc,20445
 tests/backends/test_commit.py,sha256=iTHfK1qsBxim0XwxgMvNNSMqDUMEHoYkYBDcgxGBFbs,831
@@ -75,7 +75,7 @@ tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/utils/test_concurrent.py,sha256=Gwdzym2UZ1HO3rhOSGmzxImWcLFygY8P7MXHT3Q0xTE,455
 tests/utils/test_metrics.py,sha256=bI0EtGgPokMQyEqX58i0-8zvLfxRP2nWaWr2wLMaJ_o,917
 tests/utils/test_retries.py,sha256=AxJLkXWeL9AjHv_p1n0pe8CXXJp24ZQIuYBHfNcmiz4,3075
-sentry_arroyo-2.35.0.dist-info/METADATA,sha256=KGDqYsmgTK9uhnrizYv6HorZjJZjCzFez4dBbCDdwf4,2208
-sentry_arroyo-2.35.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sentry_arroyo-2.35.0.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
-sentry_arroyo-2.35.0.dist-info/RECORD,,
+sentry_arroyo-2.37.0.dist-info/METADATA,sha256=7hyEdSMQ4c_QOMh5SIfrUoyXPgBNXwlpBUp2Yz-J8go,2208
+sentry_arroyo-2.37.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sentry_arroyo-2.37.0.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
+sentry_arroyo-2.37.0.dist-info/RECORD,,