sentry-arroyo 2.19.6__tar.gz → 2.19.8__tar.gz

{sentry_arroyo-2.19.6/sentry_arroyo.egg-info → sentry_arroyo-2.19.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sentry-arroyo
-Version: 2.19.6
+Version: 2.19.8
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.19.6 → sentry_arroyo-2.19.8}/arroyo/dlq.py

@@ -44,11 +44,12 @@ class InvalidMessage(Exception):
     """
 
     def __init__(
-        self, partition: Partition, offset: int, needs_commit: bool = True
+        self, partition: Partition, offset: int, needs_commit: bool = True, reason: Optional[str] = None,
     ) -> None:
         self.partition = partition
         self.offset = offset
         self.needs_commit = needs_commit
+        self.reason = reason
 
     @classmethod
     def from_value(cls, value: BrokerValue[Any]) -> InvalidMessage:
@@ -177,7 +178,7 @@ class DlqLimitState:
 class DlqProducer(ABC, Generic[TStrategyPayload]):
     @abstractmethod
     def produce(
-        self, value: BrokerValue[TStrategyPayload]
+        self, value: BrokerValue[TStrategyPayload], reason: Optional[str] = None
     ) -> Future[BrokerValue[TStrategyPayload]]:
         """
         Produce a message to DLQ.
@@ -201,7 +202,7 @@ class NoopDlqProducer(DlqProducer[Any]):
     """
 
     def produce(
-        self, value: BrokerValue[KafkaPayload]
+        self, value: BrokerValue[KafkaPayload], reason: Optional[str] = None,
     ) -> Future[BrokerValue[KafkaPayload]]:
         future: Future[BrokerValue[KafkaPayload]] = Future()
         future.set_running_or_notify_cancel()
@@ -229,7 +230,9 @@ class KafkaDlqProducer(DlqProducer[KafkaPayload]):
         self.__topic = topic
 
     def produce(
-        self, value: BrokerValue[KafkaPayload]
+        self,
+        value: BrokerValue[KafkaPayload],
+        reason: Optional[str] = None,
     ) -> Future[BrokerValue[KafkaPayload]]:
         value.payload.headers.append(
             ("original_partition", f"{value.partition.index}".encode("utf-8"))
@@ -353,7 +356,7 @@ class DlqPolicyWrapper(Generic[TStrategyPayload]):
             self.__dlq_policy.limit, assignment
         )
 
-    def produce(self, message: BrokerValue[TStrategyPayload]) -> None:
+    def produce(self, message: BrokerValue[TStrategyPayload], reason: Optional[str] = None) -> None:
         """
         Removes all completed futures, then appends the given future to the list.
         Blocks if the list is full. If the DLQ limit is exceeded, an exception is raised.
@@ -371,7 +374,7 @@ class DlqPolicyWrapper(Generic[TStrategyPayload]):
 
         should_accept = self.__dlq_limit_state.record_invalid_message(message)
         if should_accept:
-            future = self.__dlq_policy.producer.produce(message)
+            future = self.__dlq_policy.producer.produce(message, reason)
             self.__futures[message.partition].append((message, future))
         else:
             raise RuntimeError("Dlq limit exceeded")

{sentry_arroyo-2.19.6 → sentry_arroyo-2.19.8}/arroyo/processing/processor.py

@@ -376,7 +376,7 @@ class StreamProcessor(Generic[TStrategyPayload]):
                 ) from None
 
             # XXX: This blocks if there are more than MAX_PENDING_FUTURES in the queue.
-            self.__dlq_policy.produce(invalid_message)
+            self.__dlq_policy.produce(invalid_message, exc.reason)
 
             self.__metrics_buffer.incr_timing(
                 "arroyo.consumer.dlq.time", time.time() - start_dlq
@@ -446,10 +446,27 @@ class StreamProcessor(Generic[TStrategyPayload]):
                         self.__is_paused = True
 
                     elif self.__is_paused:
-                        # A paused consumer should still poll periodically to avoid it's partitions
-                        # getting revoked by the broker after reaching the max.poll.interval.ms
-                        # Polling a paused consumer should never yield a message.
-                        assert self.__consumer.poll(0.1) is None
+                        paused_partitions = set(self.__consumer.paused())
+                        unpaused_partitions = (
+                            set(self.__consumer.tell()) - paused_partitions
+                        )
+                        if unpaused_partitions:
+                            logger.warning(
+                                "Processor in paused state while consumer is partially unpaused: %s, paused: %s",
+                                unpaused_partitions,
+                                paused_partitions,
+                            )
+                            self.__is_paused = False
+                            # unpause paused partitions... just in case a subset is paused
+                            self.__metrics_buffer.incr_counter(
+                                "arroyo.consumer.resume", 1
+                            )
+                            self.__consumer.resume([*paused_partitions])
+                        else:
+                            # A paused consumer should still poll periodically to avoid it's partitions
+                            # getting revoked by the broker after reaching the max.poll.interval.ms
+                            # Polling a paused consumer should never yield a message.
+                            assert self.__consumer.poll(0.1) is None
                     else:
                         time.sleep(0.01)
 

{sentry_arroyo-2.19.6 → sentry_arroyo-2.19.8/sentry_arroyo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sentry-arroyo
-Version: 2.19.6
+Version: 2.19.8
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.19.6 → sentry_arroyo-2.19.8}/setup.py

@@ -10,7 +10,7 @@ def get_requirements() -> Sequence[str]:
 
 setup(
     name="sentry-arroyo",
-    version="2.19.6",
+    version="2.19.8",
     author="Sentry",
     author_email="oss@sentry.io",
     license="Apache-2.0",