flo-python 0.1.0.dev3__py3-none-any.whl → 0.1.1__py3-none-any.whl

flo/worker.py

@@ -3,7 +3,8 @@
 Provides ActionWorker for executing actions and StreamWorker for
 processing stream records via consumer groups.
 
-Example:
+Example::
+
     from flo import FloClient, ActionContext
 
     async def process_order(ctx: ActionContext) -> bytes:
@@ -14,8 +15,9 @@ Example:
     async def main():
         async with FloClient("localhost:3000", namespace="myapp") as client:
             worker = client.new_action_worker(concurrency=5)
-            worker.action("process-order")(process_order)
-            await worker.start()
+            worker.register_action("process-order", process_order)
+            async with worker:
+                await worker.start()
 """
 
 import asyncio
@@ -626,6 +628,19 @@ class ActionWorker:
         if self._client:
             await self._client.close()
 
+    async def __aenter__(self) -> "ActionWorker":
+        """Async context manager entry."""
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: object,
+    ) -> None:
+        """Async context manager exit — closes the worker."""
+        await self.close()
+
 
 # =============================================================================
 # Stream Worker
@@ -651,6 +666,15 @@ class StreamWorkerOptions:
     batch_size: int = 10
     block_ms: int = 30000
     message_timeout: float = 300.0  # 5 minutes
+    # Drain this consumer's pending (delivered-but-unacked) entries via
+    # group_claim after a reconnect, before resuming normal group_read.
+    # Required for at-least-once across reconnects: group_read alone only
+    # returns records past last_delivered_id. Set False for at-most-once.
+    redeliver_pending_on_reconnect: bool = True
+    # Minimum idle time (ms) an entry must have before the reconnect drain
+    # claims it. 0 = drain everything this consumer already owns; > 0 also
+    # steals entries abandoned by other (dead) consumers idle at least this long.
+    claim_min_idle_ms: int = 0
 
 
 @dataclass
@@ -734,6 +758,8 @@ class StreamWorker:
         batch_size: int = 10,
         block_ms: int = 30000,
         message_timeout: float = 300.0,
+        redeliver_pending_on_reconnect: bool = True,
+        claim_min_idle_ms: int = 0,
     ):
         self._parent_client = parent_client
         self._handler = handler
@@ -746,6 +772,8 @@ class StreamWorker:
             batch_size=batch_size,
             block_ms=block_ms,
             message_timeout=message_timeout,
+            redeliver_pending_on_reconnect=redeliver_pending_on_reconnect,
+            claim_min_idle_ms=claim_min_idle_ms,
         )
 
         self._client: FloClient | None = None
@@ -790,18 +818,19 @@ class StreamWorker:
         await self._client.connect()
 
         try:
-            # Join consumer group
-            await self._client.stream.group_join(
-                self.config.stream,
-                self.config.group,
-                self.config.consumer,
-            )
-            logger.info(f"Joined consumer group {self.config.group} on stream {self.config.stream}")
-
+            # Semaphore and running state must be set up before the join: a
+            # connection error during the join triggers _handle_reconnect ->
+            # _drain_pending, which acquires the semaphore.
             self._semaphore = asyncio.Semaphore(self.config.concurrency)
             self._running = True
             self._stop_event.clear()
 
+            # Join consumer group. A connection error here (a stalled or dropped
+            # connection at startup) is retried via reconnect rather than
+            # killing the worker — mirroring the poll loop and ack retry.
+            await self._join_group()
+            logger.info(f"Joined consumer group {self.config.group} on stream {self.config.stream}")
+
             await self._poll_loop()
 
         finally:
@@ -885,8 +914,41 @@ class StreamWorker:
                     logger.error(f"Stream read error: {e}, retrying...")
                     await asyncio.sleep(1)
 
+    async def _join_group(self) -> None:
+        """Join the consumer group, retrying on connection errors.
+
+        A connection error (a stalled or dropped connection at startup) triggers
+        a reconnect-and-retry instead of failing the worker. Non-connection
+        errors (e.g. an invalid request) remain fatal and are raised.
+        """
+        assert self._client is not None
+        while True:
+            if self._stop_event.is_set():
+                return
+            try:
+                await self._client.stream.group_join(
+                    self.config.stream,
+                    self.config.group,
+                    self.config.consumer,
+                )
+                return
+            except Exception as e:
+                if not is_connection_error(e):
+                    raise
+                logger.warning(f"Connection lost during group join: {e}, reconnecting...")
+                with contextlib.suppress(Exception):
+                    await self._handle_reconnect()
+                await asyncio.sleep(1)
+
     async def _handle_reconnect(self) -> None:
-        """Reconnect and re-join the consumer group."""
+        """Reconnect, re-join the consumer group, and drain pending entries.
+
+        After re-joining, this consumer's pending (delivered-but-unacked)
+        entries are re-processed via a group_claim cursor loop before normal
+        reads resume — group_read only returns records past last_delivered_id,
+        so without this an in-flight record at crash time is never re-surfaced.
+        See ``redeliver_pending_on_reconnect``.
+        """
         assert self._client is not None
         await self._client.reconnect()
         await self._client.stream.group_join(
@@ -894,6 +956,55 @@ class StreamWorker:
             self.config.group,
             self.config.consumer,
         )
+        if self.config.redeliver_pending_on_reconnect:
+            await self._drain_pending()
+
+    async def _drain_pending(self) -> None:
+        """Re-process this consumer's pending entries via a group_claim loop.
+
+        Records flow through the same _process_record path (and concurrency
+        gate) as live reads, so a handler that succeeds acks and clears the PEL
+        entry. Bounded per stream by a max page count so a pathological backlog
+        can't wedge the reconnect indefinitely; leftovers are reclaimed on the
+        next reconnect or by group_read once acked.
+        """
+        if self._semaphore is None or self._client is None:
+            # start() has not initialized the concurrency gate yet.
+            return
+
+        max_pages = 1000  # safety cap: max_pages * batch_size entries
+        cursor = StreamID()  # MIN — scan from the start of the PEL
+        pages = 0
+        while True:
+            if self._stop_event.is_set():
+                return
+            try:
+                result = await self._client.stream.group_claim(
+                    self.config.stream,
+                    self.config.group,
+                    self.config.consumer,
+                    self.config.claim_min_idle_ms,
+                    cursor,
+                    self.config.batch_size,
+                )
+            except Exception as e:
+                # Don't fail the reconnect on a drain error; normal reads resume.
+                logger.warning(f"Pending drain claim failed: {e}")
+                return
+
+            for record in result.records:
+                await self._semaphore.acquire()  # same gate as live reads
+                if self._stop_event.is_set():
+                    self._semaphore.release()
+                    return
+                task = asyncio.create_task(self._process_record(record))
+                self._tasks.add(task)
+                task.add_done_callback(self._tasks.discard)
+
+            pages += 1
+            if result.done or not result.records or pages >= max_pages:
+                return
+            cursor = result.next_cursor
 
     async def _ack_with_retry(
         self, record_ids: list[StreamID], options: StreamGroupAckOptions
@@ -977,3 +1088,16 @@ class StreamWorker:
         self.stop()
         if self._client:
             await self._client.close()
+
+    async def __aenter__(self) -> "StreamWorker":
+        """Async context manager entry."""
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: object,
+    ) -> None:
+        """Async context manager exit — closes the stream worker."""
+        await self.close()

{flo_python-0.1.0.dev3.dist-info → flo_python-0.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: flo-python
-Version: 0.1.0.dev3
+Version: 0.1.1
 Summary: Python SDK for the Flo distributed systems platform
 Project-URL: Homepage, https://github.com/floruntime/flo-python
 Project-URL: Documentation, https://github.com/floruntime/flo-python#readme
@@ -35,7 +35,7 @@ Python SDK for the [Flo](https://github.com/floruntime) distributed systems plat
 ## Installation
 
 ```bash
-pip install flo
+pip install flo-python
 ```
 
 ## Quick Start

flo_python-0.1.1.dist-info/RECORD

@@ -0,0 +1,17 @@
+flo/__init__.py,sha256=utKHOqm_iuyYkM7v8PTYynnBe2FocVd5dTdse3fdD2Q,7694
+flo/actions.py,sha256=b4cd5AAe6TRAU-_UZmenbek0h--T-C_PiiDFCHyYDTs,12258
+flo/client.py,sha256=-r4EDWYxJbcEKDWHXlMKtkJukUD7UNuI89Q0FiUTczA,16753
+flo/exceptions.py,sha256=-3HUWvRGLe_4mSWxwZXk8CMpcfrRjFv0rRkTzu4kqP4,5868
+flo/kv.py,sha256=eR9jAQrWdzvllZes4TaGvEMzvzxKJlRoe4jG89m6x8E,19429
+flo/kv_txn.py,sha256=aV9LuLkyc863OkDOk4IhuI3mfpdwShzOLx_hCR7VZgs,11906
+flo/processing.py,sha256=_jOQZiSgyArOPOdnoARNy9x6QgzcYvt1R0wa0vLQfoc,10530
+flo/queue.py,sha256=Nn6CkgUkbXgUuH9BOCWlRkjxTZLKRs0_iMyRmwkYK6k,10888
+flo/streams.py,sha256=-z5D_ZpeQbgSR1IbDFIGLqhB9TbKNISF2uQN2Sc0c-w,17168
+flo/types.py,sha256=10bqL_gm5e1-UgGeIXONLeNmcrxiGCBqOm0OUhcW4n0,31804
+flo/wire.py,sha256=grPcwuLUf2YYWNqBC_-kFj1yx9JinsJhuUerNDcwbEg,35803
+flo/worker.py,sha256=m53FWwnhGEP_iTNgwXhEJLAYvL7qoYLAJxFHr0MFSLs,40035
+flo/workflows.py,sha256=8Z5mhq2UOAnjV_Ef261mcWlj4Z5yGCwrFbxjey0ICoU,15050
+flo_python-0.1.1.dist-info/METADATA,sha256=DUzsljdTbBN7h1n6XRJlzlx4ZLEvmANaM0m1lb5WEio,14640
+flo_python-0.1.1.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
+flo_python-0.1.1.dist-info/licenses/LICENSE,sha256=mADRSuVhPxFNpZiy7zO0gBjGrNO24OtWO9VCX6wmCVU,1065
+flo_python-0.1.1.dist-info/RECORD,,

{flo_python-0.1.0.dev3.dist-info → flo_python-0.1.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.29.0
+Generator: hatchling 1.32.3
 Root-Is-Purelib: true
 Tag: py3-none-any

flo_python-0.1.0.dev3.dist-info/RECORD

@@ -1,16 +0,0 @@
-flo/__init__.py,sha256=7nQrfC6ge__E4scjMx5x-VwAiAKhsiRL-JEjC7cf0ZA,7039
-flo/actions.py,sha256=b4cd5AAe6TRAU-_UZmenbek0h--T-C_PiiDFCHyYDTs,12258
-flo/client.py,sha256=a6RpbUZD1FdInmuoZJVdwocC-vkG5efcwqevIX2_NEU,15018
-flo/exceptions.py,sha256=-3HUWvRGLe_4mSWxwZXk8CMpcfrRjFv0rRkTzu4kqP4,5868
-flo/kv.py,sha256=2PgvoZylU8Clsx8yaYdTLzdCZmD8nCZgtrmJHhAAZlM,7344
-flo/processing.py,sha256=_jOQZiSgyArOPOdnoARNy9x6QgzcYvt1R0wa0vLQfoc,10530
-flo/queue.py,sha256=Nn6CkgUkbXgUuH9BOCWlRkjxTZLKRs0_iMyRmwkYK6k,10888
-flo/streams.py,sha256=G_27-_cq6hru712olwImd4Yxuo8x4l8XuKljUk0Y1JA,12965
-flo/types.py,sha256=g8yikYiiYxsjkUlSCZxKpLMdBv7jg-cWAGO-cnEnaEo,27808
-flo/wire.py,sha256=TLCMXeA8VnUTzyyORM7G3zrsSZhyZymSGxU9jhCxmmI,29373
-flo/worker.py,sha256=AUQX5Hd6-pqm5ouCoJLUiGhRbfE6hzrACvaY0OyWNhw,34604
-flo/workflows.py,sha256=8Z5mhq2UOAnjV_Ef261mcWlj4Z5yGCwrFbxjey0ICoU,15050
-flo_python-0.1.0.dev3.dist-info/METADATA,sha256=LjTltaQ-avls3GgUlmp-ysA43IrVpx6uIwZZoVZ907E,14638
-flo_python-0.1.0.dev3.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
-flo_python-0.1.0.dev3.dist-info/licenses/LICENSE,sha256=mADRSuVhPxFNpZiy7zO0gBjGrNO24OtWO9VCX6wmCVU,1065
-flo_python-0.1.0.dev3.dist-info/RECORD,,