sentry-arroyo 2.19.8__tar.gz → 2.19.10__tar.gz

{sentry_arroyo-2.19.8/sentry_arroyo.egg-info → sentry_arroyo-2.19.10}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: sentry-arroyo
-Version: 2.19.8
+Version: 2.19.10
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry
@@ -13,7 +13,16 @@ Classifier: Programming Language :: Python
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: confluent-kafka>=2.3.0
+Requires-Dist: confluent-kafka>=2.7.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary
 
 # Arroyo
 

{sentry_arroyo-2.19.8 → sentry_arroyo-2.19.10}/arroyo/backends/kafka/consumer.py

@@ -157,13 +157,13 @@ class KafkaConsumer(Consumer[KafkaPayload]):
                 KafkaError.REQUEST_TIMED_OUT,
                 KafkaError.NOT_COORDINATOR,
                 KafkaError._WAIT_COORD,
+                KafkaError.STALE_MEMBER_EPOCH,  # kip-848
             ),
         )
 
         configuration = dict(configuration)
-        self.__is_incremental = (
+        self.__is_cooperative_sticky = (
             configuration.get("partition.assignment.strategy") == "cooperative-sticky"
-            or configuration.get("group.protocol") == "consumer"
         )
         auto_offset_reset = configuration.get("auto.offset.reset", "largest")
 
@@ -463,7 +463,7 @@ class KafkaConsumer(Consumer[KafkaPayload]):
             ConfluentTopicPartition(partition.topic.name, partition.index, offset)
             for partition, offset in offsets.items()
         ]
-        if self.__is_incremental:
+        if self.__is_cooperative_sticky:
             self.__consumer.incremental_assign(partitions)
         else:
             self.__consumer.assign(partitions)

{sentry_arroyo-2.19.8 → sentry_arroyo-2.19.10}/arroyo/backends/local/backend.py

@@ -38,9 +38,9 @@ class LocalBroker(Generic[TStrategyPayload]):
         self.__message_storage = message_storage
         self.__clock = clock
 
-        self.__offsets: MutableMapping[
-            str, MutableMapping[Partition, int]
-        ] = defaultdict(dict)
+        self.__offsets: MutableMapping[str, MutableMapping[Partition, int]] = (
+            defaultdict(dict)
+        )
 
         # The active subscriptions are stored by consumer group as a mapping
         # between the consumer and it's subscribed topics.

{sentry_arroyo-2.19.8 → sentry_arroyo-2.19.10}/arroyo/dlq.py

@@ -44,12 +44,18 @@ class InvalidMessage(Exception):
     """
 
     def __init__(
-        self, partition: Partition, offset: int, needs_commit: bool = True, reason: Optional[str] = None,
+        self,
+        partition: Partition,
+        offset: int,
+        needs_commit: bool = True,
+        reason: Optional[str] = None,
+        log_exception: bool = True,
     ) -> None:
         self.partition = partition
         self.offset = offset
         self.needs_commit = needs_commit
         self.reason = reason
+        self.log_exception = log_exception
 
     @classmethod
     def from_value(cls, value: BrokerValue[Any]) -> InvalidMessage:
@@ -202,7 +208,9 @@ class NoopDlqProducer(DlqProducer[Any]):
     """
 
     def produce(
-        self, value: BrokerValue[KafkaPayload], reason: Optional[str] = None,
+        self,
+        value: BrokerValue[KafkaPayload],
+        reason: Optional[str] = None,
     ) -> Future[BrokerValue[KafkaPayload]]:
         future: Future[BrokerValue[KafkaPayload]] = Future()
         future.set_running_or_notify_cancel()
@@ -334,7 +342,7 @@ class DlqPolicyWrapper(Generic[TStrategyPayload]):
         self,
         policy: DlqPolicy[TStrategyPayload],
     ) -> None:
-        self.MAX_PENDING_FUTURES = 1000  # This is a per partition max
+        self.MAX_PENDING_FUTURES = 2000  # This is a per partition max
         self.__dlq_policy = policy
 
         self.__futures: MutableMapping[
@@ -356,7 +364,9 @@ class DlqPolicyWrapper(Generic[TStrategyPayload]):
             self.__dlq_policy.limit, assignment
         )
 
-    def produce(self, message: BrokerValue[TStrategyPayload], reason: Optional[str] = None) -> None:
+    def produce(
+        self, message: BrokerValue[TStrategyPayload], reason: Optional[str] = None
+    ) -> None:
         """
         Removes all completed futures, then appends the given future to the list.
         Blocks if the list is full. If the DLQ limit is exceeded, an exception is raised.

{sentry_arroyo-2.19.8 → sentry_arroyo-2.19.10}/arroyo/processing/processor.py

@@ -143,9 +143,9 @@ class StreamProcessor(Generic[TStrategyPayload]):
         self.__processor_factory = processor_factory
         self.__metrics_buffer = MetricsBuffer()
 
-        self.__processing_strategy: Optional[
-            ProcessingStrategy[TStrategyPayload]
-        ] = None
+        self.__processing_strategy: Optional[ProcessingStrategy[TStrategyPayload]] = (
+            None
+        )
 
         self.__message: Optional[BrokerValue[TStrategyPayload]] = None
 
@@ -365,7 +365,8 @@ class StreamProcessor(Generic[TStrategyPayload]):
         ):
             self.__message = None
 
-        logger.exception(exc)
+        if exc.log_exception:
+            logger.exception(exc)
         self.__metrics_buffer.incr_counter("arroyo.consumer.invalid_message.count", 1)
         if self.__dlq_policy:
             start_dlq = time.time()

{sentry_arroyo-2.19.8 → sentry_arroyo-2.19.10}/arroyo/processing/strategies/filter.py

@@ -55,9 +55,9 @@ class FilterStep(ProcessingStrategy[Union[FilteredPayload, TStrategyPayload]]):
         self.__next_step = next_step
 
         if commit_policy is not None:
-            self.__commit_policy_state: Optional[
-                CommitPolicyState
-            ] = commit_policy.get_state_machine()
+            self.__commit_policy_state: Optional[CommitPolicyState] = (
+                commit_policy.get_state_machine()
+            )
         else:
             self.__commit_policy_state = None
 

{sentry_arroyo-2.19.8 → sentry_arroyo-2.19.10}/arroyo/processing/strategies/run_task_with_multiprocessing.py

@@ -625,9 +625,9 @@ class RunTaskWithMultiprocessing(
         while self.__processes:
             try:
                 self.__check_for_results_impl(
-                    timeout=max(deadline - time.time(), 0)
-                    if deadline is not None
-                    else None
+                    timeout=(
+                        max(deadline - time.time(), 0) if deadline is not None else None
+                    )
                 )
             except NextStepTimeoutError:
                 if deadline is None or deadline > time.time():

sentry_arroyo-2.19.10/requirements.txt

@@ -0,0 +1 @@
+confluent-kafka>=2.7.0

{sentry_arroyo-2.19.8 → sentry_arroyo-2.19.10/sentry_arroyo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: sentry-arroyo
-Version: 2.19.8
+Version: 2.19.10
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry
@@ -13,7 +13,16 @@ Classifier: Programming Language :: Python
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: confluent-kafka>=2.3.0
+Requires-Dist: confluent-kafka>=2.7.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary
 
 # Arroyo
 

{sentry_arroyo-2.19.8 → sentry_arroyo-2.19.10}/sentry_arroyo.egg-info/SOURCES.txt

@@ -59,6 +59,7 @@ sentry_arroyo.egg-info/requires.txt
 sentry_arroyo.egg-info/top_level.txt
 tests/test_commit.py
 tests/test_dlq.py
+tests/test_kip848_e2e.py
 tests/test_types.py
 tests/backends/__init__.py
 tests/backends/mixins.py

sentry_arroyo-2.19.10/sentry_arroyo.egg-info/requires.txt

@@ -0,0 +1 @@
+confluent-kafka>=2.7.0

{sentry_arroyo-2.19.8 → sentry_arroyo-2.19.10}/setup.py

@@ -10,7 +10,7 @@ def get_requirements() -> Sequence[str]:
 
 setup(
     name="sentry-arroyo",
-    version="2.19.8",
+    version="2.19.10",
     author="Sentry",
     author_email="oss@sentry.io",
     license="Apache-2.0",

{sentry_arroyo-2.19.8 → sentry_arroyo-2.19.10}/tests/backends/mixins.py

@@ -15,6 +15,7 @@ from tests.assertions import assert_changes, assert_does_not_change
 
 class StreamsTestMixin(ABC, Generic[TStrategyPayload]):
     cooperative_sticky = False
+    kip_848 = False
 
     @abstractmethod
     def get_topic(self, partitions: int = 1) -> ContextManager[Topic]:
@@ -421,7 +422,7 @@ class StreamsTestMixin(ABC, Generic[TStrategyPayload]):
             def wait_until_rebalancing(
                 from_consumer: Consumer[Any], to_consumer: Consumer[Any]
             ) -> None:
-                for _ in range(10):
+                for _ in range(20):
                     assert from_consumer.poll(0) is None
                     if to_consumer.poll(1.0) is not None:
                         return
@@ -453,9 +454,10 @@ class StreamsTestMixin(ABC, Generic[TStrategyPayload]):
 
             wait_until_rebalancing(consumer_a, consumer_b)
 
-            if self.cooperative_sticky:
+            if self.cooperative_sticky or self.kip_848:
                 # within incremental rebalancing, only one partition should have been reassigned to the consumer_b, and consumer_a should remain paused
-                assert consumer_a.paused() == [Partition(topic, 1)]
+                # Either partition 0 or 1 might be the paused one
+                assert len(consumer_a.paused()) == 1
                 assert consumer_a.poll(10.0) is None
             else:
                 # The first consumer should have had its offsets rolled back, as
@@ -481,20 +483,28 @@ class StreamsTestMixin(ABC, Generic[TStrategyPayload]):
 
             assert len(consumer_b.tell()) == 2
 
-            if self.cooperative_sticky:
+            if self.cooperative_sticky or self.kip_848:
+                consumer_a_on_assign.assert_has_calls(
+                    [
+                        mock.call({Partition(topic, 0): 0, Partition(topic, 1): 0}),
+                    ]
+                )
 
-                assert consumer_a_on_assign.mock_calls == [
-                    mock.call({Partition(topic, 0): 0, Partition(topic, 1): 0}),
-                ]
-                assert consumer_a_on_revoke.mock_calls == [
-                    mock.call([Partition(topic, 0)]),
-                    mock.call([Partition(topic, 1)]),
-                ]
+                consumer_a_on_revoke.assert_has_calls(
+                    [
+                        mock.call([Partition(topic, 0)]),
+                        mock.call([Partition(topic, 1)]),
+                    ],
+                    any_order=True,
+                )
 
-                assert consumer_b_on_assign.mock_calls == [
-                    mock.call({Partition(topic, 0): 0}),
-                    mock.call({Partition(topic, 1): 0}),
-                ]
+                consumer_b_on_assign.assert_has_calls(
+                    [
+                        mock.call({Partition(topic, 0): 0}),
+                        mock.call({Partition(topic, 1): 0}),
+                    ],
+                    any_order=True,
+                )
                 assert consumer_b_on_revoke.mock_calls == []
             else:
                 assert consumer_a_on_assign.mock_calls == [

{sentry_arroyo-2.19.8 → sentry_arroyo-2.19.10}/tests/backends/test_kafka.py

@@ -271,10 +271,13 @@ class TestKafkaStreamsIncrementalRebalancing(TestKafkaStreams):
     cooperative_sticky = True
 
 
-@pytest.mark.skip("kip-848 not functional yet")
 class TestKafkaStreamsKip848(TestKafkaStreams):
     kip_848 = True
 
+    @pytest.mark.xfail(reason="To be fixed")
+    def test_pause_resume_rebalancing(self) -> None:
+        super().test_pause_resume_rebalancing()
+
 
 def test_commit_codec() -> None:
     commit = Commit(

sentry_arroyo-2.19.10/tests/test_kip848_e2e.py

@@ -0,0 +1,114 @@
+from typing import Any
+
+import time
+import contextlib
+from contextlib import closing
+import os
+import threading
+import logging
+from typing import Iterator, Mapping
+
+from confluent_kafka.admin import AdminClient, NewTopic
+from arroyo.types import Commit, Message, Partition, Topic
+from arroyo.backends.kafka.configuration import build_kafka_consumer_configuration
+from arroyo.backends.kafka.consumer import KafkaConsumer, KafkaPayload
+from arroyo.processing.strategies import RunTask, CommitOffsets, ProcessingStrategy
+from arroyo.processing.strategies.abstract import ProcessingStrategyFactory
+from arroyo.processing.processor import StreamProcessor
+from arroyo.backends.kafka import KafkaProducer
+
+logging.basicConfig(level=logging.INFO)
+
+TOPIC = "test-kip848"
+
+
+@contextlib.contextmanager
+def get_topic(
+    configuration: Mapping[str, Any], partitions_count: int
+) -> Iterator[Topic]:
+    name = TOPIC
+    configuration = dict(configuration)
+    client = AdminClient(configuration)
+    [[key, future]] = client.create_topics(
+        [NewTopic(name, num_partitions=partitions_count, replication_factor=1)]
+    ).items()
+    assert key == name
+    assert future.result() is None
+    try:
+        yield Topic(name)
+    finally:
+        [[key, future]] = client.delete_topics([name]).items()
+        assert key == name
+        assert future.result() is None
+
+
+def test_kip848_e2e() -> None:
+    counter = 0
+
+    def print_msg(message: Message[Any]) -> Message[Any]:
+        nonlocal counter
+        ((partition, offset),) = message.committable.items()
+        print(f"message: {partition.index}-{offset}")
+        counter += 1
+        return message
+
+    class Strat(RunTask[Any, Any]):
+        def join(self, *args: Any, **kwargs: Any) -> None:
+            print("joining strategy, sleeping 5 seconds")
+            time.sleep(5)
+            print("joining strategy, sleeping 5 seconds -- DONE")
+            return super().join(*args, **kwargs)
+
+    class Factory(ProcessingStrategyFactory[KafkaPayload]):
+        def create_with_partitions(
+            self, commit: Commit, partitions: Mapping[Partition, int]
+        ) -> ProcessingStrategy[KafkaPayload]:
+            print("assign: ", [p.index for p in partitions])
+            return Strat(print_msg, CommitOffsets(commit))
+
+    default_config = {
+        "bootstrap.servers": os.environ.get("DEFAULT_BROKERS", "localhost:9092")
+    }
+
+    with get_topic(default_config, 2) as topic:
+        producer = KafkaProducer(default_config)
+
+        with closing(producer):
+            for i in range(30):
+                message = KafkaPayload(None, i.to_bytes(1, "big"), [])
+                producer.produce(topic, message).result()
+
+        consumer_config = build_kafka_consumer_configuration(
+            default_config,
+            group_id="kip848",
+        )
+
+        consumer_config["group.protocol"] = "consumer"
+        consumer_config.pop("session.timeout.ms", None)
+        consumer_config.pop("max.poll.interval.ms", None)
+        consumer_config.pop("partition.assignment.strategy", None)
+        consumer_config.pop("group.protocol.type", None)
+        consumer_config.pop("heartbeat.interval.ms", None)
+
+        consumer = KafkaConsumer(consumer_config)
+
+        processor = StreamProcessor(
+            consumer=consumer, topic=Topic(TOPIC), processor_factory=Factory()
+        )
+
+        def shutdown() -> None:
+            for i in range(100):
+                time.sleep(0.1)
+                if counter == 30:
+                    break
+            print("shutting down")
+            processor.signal_shutdown()
+
+        t = threading.Thread(target=shutdown)
+        t.start()
+
+        processor.run()
+
+        assert counter == 30
+
+        t.join()

sentry_arroyo-2.19.8/requirements.txt

@@ -1 +0,0 @@
-confluent-kafka>=2.3.0

sentry_arroyo-2.19.8/sentry_arroyo.egg-info/requires.txt

@@ -1 +0,0 @@
-confluent-kafka>=2.3.0