sentry-arroyo 2.31.1__py3-none-any.whl → 2.32.0__py3-none-any.whl

arroyo/backends/abstract.py

@@ -153,10 +153,13 @@ class Consumer(Generic[TStrategyPayload], ABC):
         raise NotImplementedError
 
     @abstractmethod
-    def commit_offsets(self) -> Mapping[Partition, int]:
+    def commit_offsets(self) -> Optional[Mapping[Partition, int]]:
         """
         Commit staged offsets. The return value of this method is a mapping
         of streams with their committed offsets as values.
+
+        When auto-commit is enabled (in Kafka consumers), returns None since
+        the broker handles commits automatically.
         """
         raise NotImplementedError
 

arroyo/backends/kafka/configuration.py

@@ -237,6 +237,7 @@ def build_kafka_consumer_configuration(
     bootstrap_servers: Optional[Sequence[str]] = None,
     override_params: Optional[Mapping[str, Any]] = None,
     strict_offset_reset: Optional[bool] = None,
+    enable_auto_commit: bool = False,
 ) -> KafkaBrokerConfig:
 
     if auto_offset_reset is None:
@@ -252,20 +253,23 @@ def build_kafka_consumer_configuration(
         default_config, bootstrap_servers, override_params
     )
 
-    broker_config.update(
-        {
-            "enable.auto.commit": False,
-            "enable.auto.offset.store": False,
-            "group.id": group_id,
-            "auto.offset.reset": auto_offset_reset,
-            # this is an arroyo specific flag that only affects the consumer.
-            "arroyo.strict.offset.reset": strict_offset_reset,
-            # overridden to reduce memory usage when there's a large backlog
-            "queued.max.messages.kbytes": queued_max_messages_kbytes,
-            "queued.min.messages": queued_min_messages,
-            "enable.partition.eof": False,
-            "statistics.interval.ms": STATS_COLLECTION_FREQ_MS,
-            "stats_cb": stats_callback,
-        }
-    )
+    # Default configuration with manual commit management
+    config_update = {
+        "enable.auto.commit": False,
+        "enable.auto.offset.store": False,
+        "group.id": group_id,
+        "auto.offset.reset": auto_offset_reset,
+        # this is an arroyo specific flag that only affects the consumer.
+        "arroyo.strict.offset.reset": strict_offset_reset,
+        # this is an arroyo specific flag to enable auto-commit mode
+        "arroyo.enable.auto.commit": enable_auto_commit,
+        # overridden to reduce memory usage when there's a large backlog
+        "queued.max.messages.kbytes": queued_max_messages_kbytes,
+        "queued.min.messages": queued_min_messages,
+        "enable.partition.eof": False,
+        "statistics.interval.ms": STATS_COLLECTION_FREQ_MS,
+        "stats_cb": stats_callback,
+    }
+
+    broker_config.update(config_update)
     return broker_config

arroyo/backends/kafka/consumer.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import logging
+import time
+from collections import defaultdict
 from concurrent.futures import Future
 from datetime import datetime
 from enum import Enum
@@ -21,6 +23,7 @@ from typing import (
     Tuple,
     Type,
     Union,
+    cast,
 )
 
 from confluent_kafka import (
@@ -183,6 +186,13 @@ class KafkaConsumer(Consumer[KafkaPayload]):
         if self.__strict_offset_reset is None:
             self.__strict_offset_reset = True
 
+        # Feature flag to enable rdkafka auto-commit with store_offsets
+        # When enabled, offsets are stored via store_offsets() and rdkafka
+        # automatically commits them periodically
+        self.__use_auto_commit = as_kafka_configuration_bool(
+            configuration.pop("arroyo.enable.auto.commit", False)
+        )
+
         if auto_offset_reset in {"smallest", "earliest", "beginning"}:
             self.__resolve_partition_starting_offset = (
                 self.__resolve_partition_offset_earliest
@@ -198,21 +208,32 @@ class KafkaConsumer(Consumer[KafkaPayload]):
         else:
             raise ValueError("invalid value for 'auto.offset.reset' configuration")
 
-        if (
-            as_kafka_configuration_bool(configuration.get("enable.auto.commit", "true"))
-            is not False
-        ):
-            raise ValueError("invalid value for 'enable.auto.commit' configuration")
+        # When auto-commit is disabled (default), we require explicit configuration
+        # When auto-commit is enabled, we allow rdkafka to handle commits
+        if not self.__use_auto_commit:
+            if (
+                as_kafka_configuration_bool(
+                    configuration.get("enable.auto.commit", "true")
+                )
+                is not False
+            ):
+                raise ValueError("invalid value for 'enable.auto.commit' configuration")
 
-        if (
-            as_kafka_configuration_bool(
-                configuration.get("enable.auto.offset.store", "true")
-            )
-            is not False
-        ):
-            raise ValueError(
-                "invalid value for 'enable.auto.offset.store' configuration"
-            )
+            if (
+                as_kafka_configuration_bool(
+                    configuration.get("enable.auto.offset.store", "true")
+                )
+                is not False
+            ):
+                raise ValueError(
+                    "invalid value for 'enable.auto.offset.store' configuration"
+                )
+        else:
+            # In auto-commit mode, enable auto.commit and keep auto.offset.store disabled
+            # We'll use store_offsets() manually to control which offsets get committed
+            configuration["enable.auto.commit"] = True
+            configuration["enable.auto.offset.store"] = False
+            configuration["on_commit"] = self.__on_commit_callback
 
         # NOTE: Offsets are explicitly managed as part of the assignment
         # callback, so preemptively resetting offsets is not enabled when
@@ -232,6 +253,19 @@ class KafkaConsumer(Consumer[KafkaPayload]):
 
         self.__state = KafkaConsumerState.CONSUMING
 
+    def __on_commit_callback(
+        self,
+        error: Optional[KafkaException],
+        partitions: Sequence[ConfluentTopicPartition],
+    ) -> None:
+        if error:
+            partition_info = [f"{p.topic}:{p.partition}" for p in partitions]
+            logger.warning(
+                "Commit failed: %s. Partitions: %s",
+                error,
+                partition_info,
+            )
+
     def __resolve_partition_offset_earliest(
         self, partition: ConfluentTopicPartition
     ) -> ConfluentTopicPartition:
@@ -569,7 +603,21 @@ class KafkaConsumer(Consumer[KafkaPayload]):
         # TODO: Maybe log a warning if these offsets exceed the current
         # offsets, since that's probably a side effect of an incorrect usage
         # pattern?
-        self.__staged_offsets.update(offsets)
+        if self.__use_auto_commit:
+            # When auto-commit is enabled, use store_offsets to stage offsets
+            # for rdkafka to auto-commit
+            if offsets:
+                self.__consumer.store_offsets(
+                    offsets=[
+                        ConfluentTopicPartition(
+                            partition.topic.name, partition.index, offset
+                        )
+                        for partition, offset in offsets.items()
+                    ]
+                )
+        else:
+            # Default behavior: manually track staged offsets
+            self.__staged_offsets.update(offsets)
 
     def __commit(self) -> Mapping[Partition, int]:
         if self.__state in {KafkaConsumerState.CLOSED, KafkaConsumerState.ERROR}:
@@ -617,15 +665,24 @@ class KafkaConsumer(Consumer[KafkaPayload]):
 
         return offsets
 
-    def commit_offsets(self) -> Mapping[Partition, int]:
+    def commit_offsets(self) -> Optional[Mapping[Partition, int]]:
         """
         Commit staged offsets for all partitions that this consumer is
         assigned to. The return value of this method is a mapping of
         partitions with their committed offsets as values.
 
+        When auto-commit is enabled, returns None since rdkafka handles
+        commits automatically and we don't track which offsets were committed.
+
         Raises an ``InvalidState`` if called on a closed consumer.
         """
-        return self.__commit_retry_policy.call(self.__commit)
+        if self.__use_auto_commit:
+            # When auto-commit is enabled, rdkafka commits automatically
+            # We don't track what was committed, so return None
+            # The offsets have already been staged via store_offsets()
+            return None
+        else:
+            return self.__commit_retry_policy.call(self.__commit)
 
     def close(self, timeout: Optional[float] = None) -> None:
         """
@@ -748,6 +805,9 @@ class KafkaProducer(Producer[KafkaPayload]):
 # Type alias for the delivery callback function
 DeliveryCallback = Callable[[Optional[KafkaError], ConfluentMessage], None]
 
+# Interval between metric flushes (in seconds)
+METRICS_FREQUENCY_SEC = 1.0
+
 
 class ConfluentProducer(ConfluentKafkaProducer):  # type: ignore[misc]
     """
@@ -756,8 +816,10 @@ class ConfluentProducer(ConfluentKafkaProducer):  # type: ignore[misc]
 
     def __init__(self, configuration: Mapping[str, Any]) -> None:
         super().__init__(configuration)
-        self.__metrics = get_metrics()
         self.producer_name = configuration.get("client.id") or None
+        self.__metrics = get_metrics()
+        self.__produce_counters: MutableMapping[str, int] = defaultdict(int)
+        self.__reset_metrics()
 
     def __metrics_delivery_callback(
         self,
@@ -768,16 +830,8 @@ class ConfluentProducer(ConfluentKafkaProducer):  # type: ignore[misc]
             status = "error"
         else:
             status = "success"
-
-        tags = {"status": status}
-        if self.producer_name:
-            tags["producer_name"] = self.producer_name
-
-        self.__metrics.increment(
-            "arroyo.producer.produce_status",
-            1,
-            tags=tags,
-        )
+        self.__produce_counters[status] += 1
+        self.__throttled_record()
 
     def __delivery_callback(
         self,
@@ -797,3 +851,28 @@ class ConfluentProducer(ConfluentKafkaProducer):  # type: ignore[misc]
         user_callback = callback or on_delivery
         wrapped_callback = self.__delivery_callback(user_callback)
         super().produce(*args, on_delivery=wrapped_callback, **kwargs)
+
+    def __flush_metrics(self) -> None:
+        for status, count in self.__produce_counters.items():
+            tags = {"status": status}
+            if self.producer_name:
+                tags["producer_name"] = self.producer_name
+            self.__metrics.increment(
+                name="arroyo.producer.produce_status",
+                value=count,
+                tags=tags,
+            )
+        self.__reset_metrics()
+
+    def flush(self, timeout: float = -1) -> int:
+        # Kafka producer flush should flush metrics too
+        self.__flush_metrics()
+        return cast(int, super().flush(timeout))
+
+    def __reset_metrics(self) -> None:
+        self.__produce_counters.clear()
+        self.__last_record_time = time.time()
+
+    def __throttled_record(self) -> None:
+        if time.time() - self.__last_record_time > METRICS_FREQUENCY_SEC:
+            self.__flush_metrics()

arroyo/backends/local/backend.py

@@ -38,9 +38,9 @@ class LocalBroker(Generic[TStrategyPayload]):
         self.__message_storage = message_storage
         self.__clock = clock
 
-        self.__offsets: MutableMapping[str, MutableMapping[Partition, int]] = (
-            defaultdict(dict)
-        )
+        self.__offsets: MutableMapping[
+            str, MutableMapping[Partition, int]
+        ] = defaultdict(dict)
 
         # The active subscriptions are stored by consumer group as a mapping
         # between the consumer and it's subscribed topics.
@@ -326,7 +326,7 @@ class LocalConsumer(Consumer[TStrategyPayload]):
             # atomic
             self.__staged_offsets.update(offsets)
 
-    def commit_offsets(self) -> Mapping[Partition, int]:
+    def commit_offsets(self) -> Optional[Mapping[Partition, int]]:
         with self.__lock:
             if self.__closed:
                 raise RuntimeError("consumer is closed")

arroyo/processing/processor.py

@@ -146,9 +146,9 @@ class StreamProcessor(Generic[TStrategyPayload]):
         self.__processor_factory = processor_factory
         self.__metrics_buffer = MetricsBuffer()
 
-        self.__processing_strategy: Optional[
-            ProcessingStrategy[TStrategyPayload]
-        ] = None
+        self.__processing_strategy: Optional[ProcessingStrategy[TStrategyPayload]] = (
+            None
+        )
 
         self.__message: Optional[BrokerValue[TStrategyPayload]] = None
 
@@ -465,8 +465,9 @@ class StreamProcessor(Generic[TStrategyPayload]):
 
                     elif self.__is_paused:
                         paused_partitions = set(self.__consumer.paused())
+                        all_partitions = set(self.__consumer.tell())
                         unpaused_partitions = (
-                            set(self.__consumer.tell()) - paused_partitions
+                            all_partitions - paused_partitions
                         )
                         if unpaused_partitions:
                             logger.warning(
@@ -484,6 +485,18 @@ class StreamProcessor(Generic[TStrategyPayload]):
                             # A paused consumer should still poll periodically to avoid it's partitions
                             # getting revoked by the broker after reaching the max.poll.interval.ms
                             # Polling a paused consumer should never yield a message.
+                            logger.warning("consumer.tell() value right before poll() is: %s", self.__consumer.tell())
+                            maybe_message = self.__consumer.poll(0.1)
+                            if maybe_message is not None:
+                                logger.warning("Received a message from partition: %s, \
+                                                consumer.tell() value right after poll() is: %s \
+                                                Some lines above consumer.tell() was called, all_partitons value was: %s \
+                                                Some lines above consumer.paused() was called, paused_partitions value is: %s",
+                                                maybe_message.partition,
+                                                self.__consumer.tell(),
+                                                all_partitions,
+                                                paused_partitions
+                                                )
                             assert self.__consumer.poll(0.1) is None
                     else:
                         time.sleep(0.01)

{sentry_arroyo-2.31.1.dist-info → sentry_arroyo-2.32.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sentry-arroyo
-Version: 2.31.1
+Version: 2.32.0
 Summary: Arroyo is a Python library for working with streaming data.
 Home-page: https://github.com/getsentry/arroyo
 Author: Sentry

{sentry_arroyo-2.31.1.dist-info → sentry_arroyo-2.32.0.dist-info}/RECORD

@@ -5,18 +5,18 @@ arroyo/errors.py,sha256=IbtoIbz_m5QrxNRBLOxiy-hOfJQTEwNPCyq6yqedJYk,1059
 arroyo/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 arroyo/types.py,sha256=sLY0x030np4UmbaW5C1KH1se7Z2pjQiPvAe5x2sXf7A,5684
 arroyo/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arroyo/backends/abstract.py,sha256=NQ5xG9rjchSUW8URl1WtSYSzMFtwRKB6wtJhWmDyR9E,9386
+arroyo/backends/abstract.py,sha256=Wy9xhE1dtFiumG8Cz3JhksJ0rF74uJWZWq10UO1rxOI,9524
 arroyo/backends/kafka/__init__.py,sha256=xgf-AqHbQkJsh73YokO2uoyyHfZf8XwUp6BULtM8stI,445
 arroyo/backends/kafka/commit.py,sha256=LPsjvX5PPXR62DT6sa5GuSF78qk9F_L--Fz4kw7-m-s,3060
-arroyo/backends/kafka/configuration.py,sha256=g1Y-vdu3OT9pqWuYRp9fw29Nmm1KBBExQLr3VhDsp90,8950
-arroyo/backends/kafka/consumer.py,sha256=QrhgcTiJbwZ3rf_nOn9PQL9n8L9chyXmtTGPRRiBxoY,30130
+arroyo/backends/kafka/configuration.py,sha256=zB54w7qsyVeMVkH5MpV6F8ztXfEzIXrex6aKYX-GcqA,9141
+arroyo/backends/kafka/consumer.py,sha256=zZ2ZoDaurLDBN9l9QR0fFWL16RJcf0D8Apaa3aff22k,33534
 arroyo/backends/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arroyo/backends/local/backend.py,sha256=7odjCnzoGgdo8JHLgG1ntaXa-ZR9GteGkquiA2WAWmM,13880
+arroyo/backends/local/backend.py,sha256=hUXdCV6B5e7s4mjFC6HnIuUhjENU2tNZt5vuEOJmGZQ,13888
 arroyo/backends/local/storages/__init__.py,sha256=AGYujdAAcn3osoj9jq84IzTywYbkIDv9wRg2rLhLXeg,104
 arroyo/backends/local/storages/abstract.py,sha256=1qVQp6roxHkK6XT2aklZyZk1qq7RzcPN6Db_CA5--kg,2901
 arroyo/backends/local/storages/memory.py,sha256=AoKDsVZzBXkOJyWArKWp3vfGfU9xLlKFXE9gsJiMIzQ,2613
 arroyo/processing/__init__.py,sha256=vZVg0wJvJfoVzlzGvnL59bT6YNIRJNQ5t7oU045Qbk4,87
-arroyo/processing/processor.py,sha256=WRQUnakUWctWlEcBW1s_f-xbd3dPOJ1UCOsvVYlfvLQ,21321
+arroyo/processing/processor.py,sha256=BtNaIxBApuUAtSH-syGJnpeKADHUafut9Ve1KMe8JM0,22389
 arroyo/processing/strategies/__init__.py,sha256=EU_JMb54eOxMxaC5mIFpI-sAF-X2ZScbE8czBZ7bQkY,1106
 arroyo/processing/strategies/abstract.py,sha256=nu7juEz_aQmQIH35Z8u--FBuLjkK8_LQ1hIG2xpw9AA,4808
 arroyo/processing/strategies/batching.py,sha256=s89xC6lQpBseEaApu1iNTipXGKeO95OMwinj2VBKn9s,4778
@@ -46,12 +46,12 @@ examples/transform_and_produce/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 examples/transform_and_produce/batched.py,sha256=st2R6qTneAtV0JFbKP30Ti3sJDYj8Jkbmta9JckKdZU,2636
 examples/transform_and_produce/script.py,sha256=8kSMIjQNqGYEVyE0PvrfJh-a_UYCrJSstTp_De7kyyg,2306
 examples/transform_and_produce/simple.py,sha256=H7xqxItjl4tx34wVW5dy6mB9G39QucAtxkJSBzVmjgA,1637
-sentry_arroyo-2.31.1.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
+sentry_arroyo-2.32.0.dist-info/licenses/LICENSE,sha256=0Ng3MFdEcnz0sVD1XvGBBzbavvNp_7OAM5yVObB46jU,10829
 tests/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/backends/mixins.py,sha256=sfNyE0VTeiD3GHOnBYl-9urvPuURI2G1BWke0cz7Dvc,20445
 tests/backends/test_commit.py,sha256=iTHfK1qsBxim0XwxgMvNNSMqDUMEHoYkYBDcgxGBFbs,831
-tests/backends/test_confluent_producer.py,sha256=zMCRqXdrWSgkwIspZJ7uhzpkQIFZKNbQeMAG1BeQPms,2993
-tests/backends/test_kafka.py,sha256=6W6EA41X-ECTfcOeivhQxURnmV2Y0fYy-UzDCnJgDsU,12830
+tests/backends/test_confluent_producer.py,sha256=KWqgvjDvqAdd0HxngdWKsUJaV7Hl1L5vAVQhBYlHeHU,3146
+tests/backends/test_kafka.py,sha256=wBFCKEHoP6h0uG1bgDuzk84IZmrV_UVOFCrtbxztmJg,15506
 tests/backends/test_kafka_producer.py,sha256=LpwkqnstcCDxemlKZ0FpzNKrP-1UuXXY15P7P-spjhE,3912
 tests/backends/test_local.py,sha256=Mfd4DFuWVSVtl1GomQ6TIoWuJNcAliKqKU0BShPlEMY,3363
 tests/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -74,7 +74,7 @@ tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/utils/test_concurrent.py,sha256=Gwdzym2UZ1HO3rhOSGmzxImWcLFygY8P7MXHT3Q0xTE,455
 tests/utils/test_metrics.py,sha256=bI0EtGgPokMQyEqX58i0-8zvLfxRP2nWaWr2wLMaJ_o,917
 tests/utils/test_retries.py,sha256=AxJLkXWeL9AjHv_p1n0pe8CXXJp24ZQIuYBHfNcmiz4,3075
-sentry_arroyo-2.31.1.dist-info/METADATA,sha256=JREC1Nm6b51e0nmz9ANO0N86Qh2th772Ri8t-I5FXJc,2208
-sentry_arroyo-2.31.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sentry_arroyo-2.31.1.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
-sentry_arroyo-2.31.1.dist-info/RECORD,,
+sentry_arroyo-2.32.0.dist-info/METADATA,sha256=mzNqLp5VxLgwXIgYJQUGelVVtP_Vy5RCCCrNiOEk2FU,2208
+sentry_arroyo-2.32.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sentry_arroyo-2.32.0.dist-info/top_level.txt,sha256=DVdMZKysL_iIxm5aY0sYgZtP5ZXMg9YBaBmGQHVmDXA,22
+sentry_arroyo-2.32.0.dist-info/RECORD,,

tests/backends/test_confluent_producer.py

@@ -29,6 +29,7 @@ class TestConfluentProducer:
         )
         mock_message = mock.Mock(spec=ConfluentMessage)
         producer._ConfluentProducer__metrics_delivery_callback(None, mock_message)
+        producer.flush()  # Flush buffered metrics
         assert (
             Increment(
                 "arroyo.producer.produce_status",
@@ -44,6 +45,7 @@ class TestConfluentProducer:
         mock_error = mock.Mock(spec=KafkaError)
         mock_message = mock.Mock(spec=ConfluentMessage)
         producer._ConfluentProducer__metrics_delivery_callback(mock_error, mock_message)
+        producer.flush()  # Flush buffered metrics
         assert (
             Increment("arroyo.producer.produce_status", 1, {"status": "error"})
             in TestingMetricsBackend.calls
@@ -64,6 +66,7 @@ class TestConfluentProducer:
         wrapped = producer._ConfluentProducer__delivery_callback(user_callback)
         mock_message = mock.Mock(spec=ConfluentMessage)
         wrapped(None, mock_message)
+        producer.flush()  # Flush buffered metrics
         assert (
             Increment(
                 "arroyo.producer.produce_status",

tests/backends/test_kafka.py

@@ -275,6 +275,70 @@ class TestKafkaStreams(StreamsTestMixin[KafkaPayload]):
                 processor._run_once()
                 assert consumer.paused() == []
 
+    def test_auto_commit_mode(self) -> None:
+        """Test that auto-commit mode uses store_offsets and commits on close"""
+        group_id = uuid.uuid1().hex
+
+        with self.get_topic() as topic:
+            # Produce some messages
+            with closing(self.get_producer()) as producer:
+                for i in range(5):
+                    payload = KafkaPayload(None, f"msg_{i}".encode("utf8"), [])
+                    producer.produce(topic, payload).result(5.0)
+
+            # Create consumer with auto-commit enabled
+            configuration = {
+                **self.configuration,
+                "auto.offset.reset": "earliest",
+                "arroyo.enable.auto.commit": True,
+                "group.id": group_id,
+                "session.timeout.ms": 10000,
+            }
+
+            # First consumer: consume messages and close
+            consumed_offsets = []
+            with closing(KafkaConsumer(configuration)) as consumer:
+                consumer.subscribe([topic])
+
+                # Consume all 5 messages and stage their offsets
+                for i in range(5):
+                    value = consumer.poll(10.0)
+                    assert value is not None
+                    consumed_offsets.append(value.offset)
+
+                    # Stage offsets (will use store_offsets internally in auto-commit mode)
+                    consumer.stage_offsets(value.committable)
+
+                # commit_offsets should return None in auto-commit mode
+                result = consumer.commit_offsets()
+                assert result is None
+
+                # Close will commit any stored offsets
+
+            # Verify we consumed offsets 0-4
+            assert consumed_offsets == [0, 1, 2, 3, 4]
+
+            # Second consumer: verify offsets were committed on close
+            # This consumer uses manual commit to verify the committed offset
+            with closing(
+                self.get_consumer(
+                    group=group_id,
+                    auto_offset_reset="earliest",
+                    enable_end_of_partition=True,
+                )
+            ) as consumer:
+                consumer.subscribe([topic])
+
+                # Should start from offset 5, hitting EndOfPartition immediately
+                # If we got a message with offset < 5, auto-commit didn't work
+                try:
+                    consumer.poll(10.0)
+                    pytest.fail("Expected EndOfPartition, but poll succeeded")
+                except EndOfPartition as e:
+                    # Verify we got EndOfPartition at offset 5
+                    assert e.offset == 5
+                    assert e.partition == Partition(topic, 0)
+
 
 class TestKafkaStreamsIncrementalRebalancing(TestKafkaStreams):
     # re-test the kafka consumer with cooperative-sticky rebalancing