sentry-arroyo 2.20.5__tar.gz → 2.20.7__tar.gz

{sentry_arroyo-2.20.5/sentry_arroyo.egg-info → sentry_arroyo-2.20.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.20.5
+Version: 2.20.7
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.20.5 → sentry_arroyo-2.20.7}/arroyo/backends/abstract.py

@@ -1,12 +1,24 @@
 from __future__ import annotations
 
 import logging
+import time
 from abc import ABC, abstractmethod, abstractproperty
 from concurrent.futures import Future
-from typing import Callable, Generic, Mapping, Optional, Sequence, Union
+from typing import (
+    Callable,
+    Generic,
+    Mapping,
+    Optional,
+    Protocol,
+    Sequence,
+    TypeVar,
+    Union,
+)
 
 from arroyo.types import BrokerValue, Partition, Topic, TStrategyPayload
 
+T = TypeVar("T")
+
 logger = logging.getLogger(__name__)
 
 
@@ -175,11 +187,72 @@ class Consumer(Generic[TStrategyPayload], ABC):
         raise NotImplementedError
 
 
+class ProducerFuture(Protocol, Generic[T]):
+    """
+    An abstract interface for a kind of Future. Stdlib futures are too slow to
+    construct, so we use these.
+    """
+
+    def done(self) -> bool:
+        ...
+
+    def result(self, timeout: float | None = None) -> T:
+        """
+        Return result or raise exception. May block, but does not have to.
+        """
+        ...
+
+    def set_result(self, result: T) -> None:
+        ...
+
+    def set_exception(self, exception: Exception) -> None:
+        ...
+
+
+class SimpleProducerFuture(Generic[T]):
+    """
+    A stub for concurrent.futures.Future that does not construct any Condition
+    variables, therefore is faster to construct.
+    """
+
+    def __init__(self) -> None:
+        self.result_value: T | None = None
+        self.result_exception: Exception | None = None
+
+    def done(self) -> bool:
+        return self.result_value is not None or self.result_exception is not None
+
+    def result(self, timeout: float | None = None) -> T:
+        if timeout is not None:
+            deadline = time.time() + timeout
+        else:
+            deadline = None
+
+        # This implementation is bogus and shouldn't be used in production,
+        # only in tests at most. It is only here for the sake of implementing
+        # the contract. If you really need result with timeout>0, you should
+        # use the stdlib future.
+        while deadline is None or time.time() < deadline:
+            if self.result_exception is not None:
+                raise self.result_exception
+            if self.result_value is not None:
+                return self.result_value
+            time.sleep(0.1)
+
+        raise TimeoutError()
+
+    def set_result(self, result: T) -> None:
+        self.result_value = result
+
+    def set_exception(self, exception: Exception) -> None:
+        self.result_exception = exception
+
+
 class Producer(Generic[TStrategyPayload], ABC):
     @abstractmethod
     def produce(
         self, destination: Union[Topic, Partition], payload: TStrategyPayload
-    ) -> Future[BrokerValue[TStrategyPayload]]:
+    ) -> ProducerFuture[BrokerValue[TStrategyPayload]]:
         """
         Produce to a topic or partition.
         """

{sentry_arroyo-2.20.5 → sentry_arroyo-2.20.7}/arroyo/backends/kafka/consumer.py

@@ -36,7 +36,12 @@ from confluent_kafka import Message as ConfluentMessage
 from confluent_kafka import Producer as ConfluentProducer
 from confluent_kafka import TopicPartition as ConfluentTopicPartition
 
-from arroyo.backends.abstract import Consumer, Producer
+from arroyo.backends.abstract import (
+    Consumer,
+    Producer,
+    ProducerFuture,
+    SimpleProducerFuture,
+)
 from arroyo.errors import (
     ConsumerError,
     EndOfPartition,
@@ -645,7 +650,9 @@ class KafkaConsumer(Consumer[KafkaPayload]):
 
 
 class KafkaProducer(Producer[KafkaPayload]):
-    def __init__(self, configuration: Mapping[str, Any]) -> None:
+    def __init__(
+        self, configuration: Mapping[str, Any], use_simple_futures: bool = False
+    ) -> None:
         self.__configuration = configuration
 
         self.__producer = ConfluentProducer(configuration)
@@ -656,6 +663,8 @@ class KafkaProducer(Producer[KafkaPayload]):
         # ``produce(...).result()`` could result in a deadlock.
         self.__result = execute(self.__worker, daemon=True)
 
+        self.use_simple_futures = use_simple_futures
+
     def __worker(self) -> None:
         """
         Continuously polls the producer to ensure that delivery callbacks are
@@ -670,7 +679,7 @@ class KafkaProducer(Producer[KafkaPayload]):
 
     def __delivery_callback(
         self,
-        future: Future[BrokerValue[KafkaPayload]],
+        future: ProducerFuture[BrokerValue[KafkaPayload]],
         payload: KafkaPayload,
         error: KafkaError,
         message: ConfluentMessage,
@@ -698,7 +707,7 @@ class KafkaProducer(Producer[KafkaPayload]):
         self,
         destination: Union[Topic, Partition],
         payload: KafkaPayload,
-    ) -> Future[BrokerValue[KafkaPayload]]:
+    ) -> ProducerFuture[BrokerValue[KafkaPayload]]:
         if self.__shutdown_requested.is_set():
             raise RuntimeError("producer has been closed")
 
@@ -713,8 +722,13 @@ class KafkaProducer(Producer[KafkaPayload]):
         else:
             raise TypeError("invalid destination type")
 
-        future: Future[BrokerValue[KafkaPayload]] = Future()
-        future.set_running_or_notify_cancel()
+        future: ProducerFuture[BrokerValue[KafkaPayload]]
+        if self.use_simple_futures:
+            future = SimpleProducerFuture()
+        else:
+            future = Future()
+            future.set_running_or_notify_cancel()
+
         produce(
             value=payload.value,
             key=payload.key,

{sentry_arroyo-2.20.5 → sentry_arroyo-2.20.7}/arroyo/dlq.py

@@ -16,7 +16,7 @@ from typing import (
     Tuple,
 )
 
-from arroyo.backends.abstract import Producer
+from arroyo.backends.abstract import Producer, ProducerFuture
 from arroyo.backends.kafka import KafkaPayload
 from arroyo.types import (
     FILTERED_PAYLOAD,
@@ -186,7 +186,7 @@ class DlqProducer(ABC, Generic[TStrategyPayload]):
     @abstractmethod
     def produce(
         self, value: BrokerValue[TStrategyPayload], reason: Optional[str] = None
-    ) -> Future[BrokerValue[TStrategyPayload]]:
+    ) -> ProducerFuture[BrokerValue[TStrategyPayload]]:
         """
         Produce a message to DLQ.
         """
@@ -212,7 +212,7 @@ class NoopDlqProducer(DlqProducer[Any]):
         self,
         value: BrokerValue[KafkaPayload],
         reason: Optional[str] = None,
-    ) -> Future[BrokerValue[KafkaPayload]]:
+    ) -> ProducerFuture[BrokerValue[KafkaPayload]]:
         future: Future[BrokerValue[KafkaPayload]] = Future()
         future.set_running_or_notify_cancel()
         future.set_result(value)
@@ -242,7 +242,7 @@ class KafkaDlqProducer(DlqProducer[KafkaPayload]):
         self,
         value: BrokerValue[KafkaPayload],
         reason: Optional[str] = None,
-    ) -> Future[BrokerValue[KafkaPayload]]:
+    ) -> ProducerFuture[BrokerValue[KafkaPayload]]:
         value.payload.headers.append(
             ("original_partition", f"{value.partition.index}".encode("utf-8"))
         )
@@ -381,7 +381,7 @@ class DlqPolicyWrapper(Generic[TStrategyPayload]):
             Deque[
                 Tuple[
                     BrokerValue[TStrategyPayload],
-                    Future[BrokerValue[TStrategyPayload]],
+                    ProducerFuture[BrokerValue[TStrategyPayload]],
                 ]
             ],
         ] = defaultdict(deque)

{sentry_arroyo-2.20.5 → sentry_arroyo-2.20.7}/arroyo/processing/processor.py

@@ -144,9 +144,9 @@ class StreamProcessor(Generic[TStrategyPayload]):
         self.__processor_factory = processor_factory
         self.__metrics_buffer = MetricsBuffer()
 
-        self.__processing_strategy: Optional[ProcessingStrategy[TStrategyPayload]] = (
-            None
-        )
+        self.__processing_strategy: Optional[
+            ProcessingStrategy[TStrategyPayload]
+        ] = None
 
         self.__message: Optional[BrokerValue[TStrategyPayload]] = None
 
@@ -381,8 +381,12 @@ class StreamProcessor(Generic[TStrategyPayload]):
             try:
                 self.__dlq_policy.produce(invalid_message, exc.reason)
             except Exception:
-                logger.exception(f"Failed to produce message (partition: {exc.partition} offset: {exc.offset}) to DLQ topic, dropping")
-                self.__metrics_buffer.incr_counter("arroyo.consumer.dlq.dropped_messages", 1)
+                logger.exception(
+                    f"Failed to produce message (partition: {exc.partition} offset: {exc.offset}) to DLQ topic, dropping"
+                )
+                self.__metrics_buffer.incr_counter(
+                    "arroyo.consumer.dlq.dropped_messages", 1
+                )
 
             self.__metrics_buffer.incr_timing(
                 "arroyo.consumer.dlq.time", time.time() - start_dlq

{sentry_arroyo-2.20.5 → sentry_arroyo-2.20.7}/arroyo/processing/strategies/produce.py

@@ -1,10 +1,9 @@
 import logging
 import time
 from collections import deque
-from concurrent.futures import Future
 from typing import Deque, Optional, Tuple, Union
 
-from arroyo.backends.abstract import Producer
+from arroyo.backends.abstract import Producer, ProducerFuture
 from arroyo.processing.strategies.abstract import MessageRejected, ProcessingStrategy
 from arroyo.types import (
     BrokerValue,
@@ -52,7 +51,7 @@ class Produce(ProcessingStrategy[Union[FilteredPayload, TStrategyPayload]]):
         self.__queue: Deque[
             Tuple[
                 Message[Union[FilteredPayload, TStrategyPayload]],
-                Optional[Future[BrokerValue[TStrategyPayload]]],
+                Optional[ProducerFuture[BrokerValue[TStrategyPayload]]],
             ]
         ] = deque()
 
@@ -92,7 +91,7 @@ class Produce(ProcessingStrategy[Union[FilteredPayload, TStrategyPayload]]):
         if len(self.__queue) >= self.__max_buffer_size:
             raise MessageRejected
 
-        future: Optional[Future[BrokerValue[TStrategyPayload]]] = None
+        future: Optional[ProducerFuture[BrokerValue[TStrategyPayload]]] = None
 
         if not isinstance(message.payload, FilteredPayload):
             try:

{sentry_arroyo-2.20.5 → sentry_arroyo-2.20.7}/arroyo/processing/strategies/unfold.py

@@ -94,7 +94,7 @@ class Unfold(
                 self.__next_step.submit(message)
                 self.__pending.popleft()
             except MessageRejected:
-                pass
+                break
 
     def poll(self) -> None:
         self.__flush()

{sentry_arroyo-2.20.5 → sentry_arroyo-2.20.7/sentry_arroyo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.20.5
+Version: 2.20.7
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.20.5 → sentry_arroyo-2.20.7}/setup.py

@@ -10,7 +10,7 @@ def get_requirements() -> Sequence[str]:
 
 setup(
     name="sentry-arroyo",
-    version="2.20.5",
+    version="2.20.7",
     author="Sentry",
     author_email="oss@sentry.io",
     license="Apache-2.0",

{sentry_arroyo-2.20.5 → sentry_arroyo-2.20.7}/tests/backends/test_kafka.py

@@ -131,8 +131,8 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
 
         return KafkaConsumer(configuration)
 
-    def get_producer(self) -> KafkaProducer:
-        return KafkaProducer(self.configuration)
+    def get_producer(self, use_simple_futures: bool = False) -> KafkaProducer:
+        return KafkaProducer(self.configuration, use_simple_futures=use_simple_futures)
 
     def get_payloads(self) -> Iterator[KafkaPayload]:
         for i in itertools.count():
@@ -150,9 +150,10 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
                 assert isinstance(value, BrokerValue)
                 assert value.payload.value == b"0"
 
-    def test_auto_offset_reset_latest(self) -> None:
+    @pytest.mark.parametrize("use_simple_futures", [True, False])
+    def test_auto_offset_reset_latest(self, use_simple_futures: bool) -> None:
         with self.get_topic() as topic:
-            with closing(self.get_producer()) as producer:
+            with closing(self.get_producer(use_simple_futures)) as producer:
                 producer.produce(topic, next(self.get_payloads())).result(5.0)
 
             with closing(self.get_consumer(auto_offset_reset="latest")) as consumer:

{sentry_arroyo-2.20.5 → sentry_arroyo-2.20.7}/tests/processing/strategies/test_unfold.py

@@ -49,6 +49,13 @@ def test_message_rejected() -> None:
         call(Message(Value(0, {}, NOW))),
     ]
 
+    # poll again, to show that it does not block (regression)
+    strategy.poll()
+    assert next_step.submit.call_args_list == [
+        call(Message(Value(0, {}, NOW))),
+        call(Message(Value(0, {}, NOW))),
+    ]
+
     # clear the side effect, both messages should be submitted now
     next_step.submit.reset_mock(side_effect=True)
 

{sentry_arroyo-2.20.5 → sentry_arroyo-2.20.7}/tests/processing/test_processor.py

@@ -583,7 +583,6 @@ def test_dlq() -> None:
     assert dlq_policy.producer.produce.call_count == 1
 
 
-
 def test_healthcheck(tmpdir: py.path.local) -> None:
     """
     Test healthcheck strategy e2e with StreamProcessor, to ensure the