sentry-arroyo 2.19.7__py3-none-any.whl → 2.19.9__py3-none-any.whl

arroyo/dlq.py

@@ -44,12 +44,18 @@ class InvalidMessage(Exception):
     """
 
     def __init__(
-        self, partition: Partition, offset: int, needs_commit: bool = True, reason: Optional[str] = None,
+        self,
+        partition: Partition,
+        offset: int,
+        needs_commit: bool = True,
+        reason: Optional[str] = None,
+        log_exception: bool = True
     ) -> None:
         self.partition = partition
         self.offset = offset
         self.needs_commit = needs_commit
         self.reason = reason
+        self.log_exception = log_exception
 
     @classmethod
     def from_value(cls, value: BrokerValue[Any]) -> InvalidMessage:

arroyo/processing/processor.py

@@ -365,7 +365,8 @@ class StreamProcessor(Generic[TStrategyPayload]):
         ):
             self.__message = None
 
-        logger.exception(exc)
+        if exc.log_exception:
+            logger.exception(exc)
         self.__metrics_buffer.incr_counter("arroyo.consumer.invalid_message.count", 1)
         if self.__dlq_policy:
             start_dlq = time.time()
@@ -446,10 +447,27 @@ class StreamProcessor(Generic[TStrategyPayload]):
                         self.__is_paused = True
 
                     elif self.__is_paused:
-                        # A paused consumer should still poll periodically to avoid it's partitions
-                        # getting revoked by the broker after reaching the max.poll.interval.ms
-                        # Polling a paused consumer should never yield a message.
-                        assert self.__consumer.poll(0.1) is None
+                        paused_partitions = set(self.__consumer.paused())
+                        unpaused_partitions = (
+                            set(self.__consumer.tell()) - paused_partitions
+                        )
+                        if unpaused_partitions:
+                            logger.warning(
+                                "Processor in paused state while consumer is partially unpaused: %s, paused: %s",
+                                unpaused_partitions,
+                                paused_partitions,
+                            )
+                            self.__is_paused = False
+                            # unpause paused partitions... just in case a subset is paused
+                            self.__metrics_buffer.incr_counter(
+                                "arroyo.consumer.resume", 1
+                            )
+                            self.__consumer.resume([*paused_partitions])
+                        else:
+                            # A paused consumer should still poll periodically to avoid it's partitions
+                            # getting revoked by the broker after reaching the max.poll.interval.ms
+                            # Polling a paused consumer should never yield a message.
+                            assert self.__consumer.poll(0.1) is None
                     else:
                         time.sleep(0.01)
 

{sentry_arroyo-2.19.7.dist-info → sentry_arroyo-2.19.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sentry-arroyo
-Version: 2.19.7
+Version: 2.19.9
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.19.7.dist-info → sentry_arroyo-2.19.9.dist-info}/RECORD

@@ -1,6 +1,6 @@
 arroyo/__init__.py,sha256=fcpHZd2P3MxWl6PJJ8n__fM_NRIfiUE8tKN-orv6lb0,187
 arroyo/commit.py,sha256=oFihWUW8fLsjomWh0o085qIHe9vwVNgoOJC6JQdFM7M,2235
-arroyo/dlq.py,sha256=EMqjhC3tHBGcFNkyu6RFGQstYwM9Zvtd9Wz5oUwpZlM,15521
+arroyo/dlq.py,sha256=BDmtJXrUKc8Pff7_AkRLE7accyyDkzakHe3mRPGJkNk,15631
 arroyo/errors.py,sha256=IbtoIbz_m5QrxNRBLOxiy-hOfJQTEwNPCyq6yqedJYk,1059
 arroyo/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/types.py,sha256=sLY0x030np4UmbaW5C1KH1se7Z2pjQiPvAe5x2sXf7A,5684
@@ -16,7 +16,7 @@ arroyo/backends/local/storages/__init__.py,sha256=AGYujdAAcn3osoj9jq84IzTywYbkID
 arroyo/backends/local/storages/abstract.py,sha256=1qVQp6roxHkK6XT2aklZyZk1qq7RzcPN6Db_CA5--kg,2901
 arroyo/backends/local/storages/memory.py,sha256=AoKDsVZzBXkOJyWArKWp3vfGfU9xLlKFXE9gsJiMIzQ,2613
 arroyo/processing/__init__.py,sha256=vZVg0wJvJfoVzlzGvnL59bT6YNIRJNQ5t7oU045Qbk4,87
-arroyo/processing/processor.py,sha256=JtBWLNrFBNG9T174VIXHHW02gK5YJaJt4f5XoNDd_lU,19010
+arroyo/processing/processor.py,sha256=RRLp8qcEL04TSvHaGuF0nc6RQ-HOiBzzI8nUwV5hZ2M,20025
 arroyo/processing/strategies/__init__.py,sha256=EU_JMb54eOxMxaC5mIFpI-sAF-X2ZScbE8czBZ7bQkY,1106
 arroyo/processing/strategies/abstract.py,sha256=nu7juEz_aQmQIH35Z8u--FBuLjkK8_LQ1hIG2xpw9AA,4808
 arroyo/processing/strategies/batching.py,sha256=s89xC6lQpBseEaApu1iNTipXGKeO95OMwinj2VBKn9s,4778
@@ -71,8 +71,8 @@ tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/utils/test_concurrent.py,sha256=Gwdzym2UZ1HO3rhOSGmzxImWcLFygY8P7MXHT3Q0xTE,455
 tests/utils/test_metrics.py,sha256=bI0EtGgPokMQyEqX58i0-8zvLfxRP2nWaWr2wLMaJ_o,917
 tests/utils/test_retries.py,sha256=AxJLkXWeL9AjHv_p1n0pe8CXXJp24ZQIuYBHfNcmiz4,3075
-sentry_arroyo-2.19.7.dist-info/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
-sentry_arroyo-2.19.7.dist-info/METADATA,sha256=-NLUjXgKgabXzEOBUpgYTxnkme0tmeEBF0OvDuGOqas,1989
-sentry_arroyo-2.19.7.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-sentry_arroyo-2.19.7.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
-sentry_arroyo-2.19.7.dist-info/RECORD,,
+sentry_arroyo-2.19.9.dist-info/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
+sentry_arroyo-2.19.9.dist-info/METADATA,sha256=wmVhRiE5W4tvuCT3CCuAYsdi6q0ZjTPq8NxhA7zCuKA,1989
+sentry_arroyo-2.19.9.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+sentry_arroyo-2.19.9.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
+sentry_arroyo-2.19.9.dist-info/RECORD,,