flo-python 0.1.0.dev4__py3-none-any.whl → 0.1.1__py3-none-any.whl

flo/types.py

@@ -6,7 +6,7 @@ Core types, constants, and data classes for the Flo client SDK.
 import struct
 from dataclasses import dataclass
 from enum import IntEnum
-from typing import Optional
+from typing import Any, Generic, Optional, TypeVar
 
 # =============================================================================
 # Protocol Constants
@@ -67,7 +67,7 @@ class OpCode(IntEnum):
     CLUSTER_MEMBERS_RESPONSE = 0x041
     CLUSTER_JOIN_RESPONSE = 0x042
 
-    # ── KV + Transactions + Snapshots (0x100 – 0x12F) ──
+    # ── KV (0x100 – 0x12F) ──
     KV_PUT = 0x100
     KV_GET = 0x101
     KV_MGET = 0x102
@@ -79,13 +79,23 @@ class OpCode(IntEnum):
     KV_PUT_RESPONSE = 0x108
     KV_SCAN_RESPONSE = 0x109
     KV_HISTORY_RESPONSE = 0x10A
+    # KV Extended (atomic counters, JSON ops)
+    KV_INCR = 0x10B
+    KV_JSON_GET = 0x10C
+    KV_JSON_SET = 0x10D
+    KV_JSON_DEL = 0x10E
+    # KV Per-Shard Transactions
     KV_BEGIN_TXN = 0x110
     KV_COMMIT_TXN = 0x111
     KV_ROLLBACK_TXN = 0x112
-    KV_SNAPSHOT_CREATE = 0x120
-    KV_SNAPSHOT_GET = 0x121
-    KV_SNAPSHOT_RELEASE = 0x122
-    KV_SNAPSHOT_CREATE_RESPONSE = 0x123
+    # KV Extended (TTL lifecycle, exists)
+    KV_TOUCH = 0x113
+    KV_PERSIST = 0x114
+    KV_EXISTS = 0x115
+    KV_INCR_RESPONSE = 0x116
+    KV_JSON_RESPONSE = 0x117
+    KV_EXISTS_RESPONSE = 0x118
+    KV_TXN_RESPONSE = 0x119
 
     # ── Streams (0x130 – 0x14F) ──
     STREAM_APPEND = 0x130
@@ -285,6 +295,7 @@ class OptionTag(IntEnum):
     KEYS_ONLY = 0x06  # u8: Skip values in scan response (0/1)
     CURSOR = 0x07  # bytes: Pagination cursor (ShardWalker format)
     ROUTING_KEY = 0x08  # string: Explicit routing key for shard co-location
+    TXN_ID = 0x09  # u64: Transaction ID for per-shard transactions
 
     # Queue Options (0x10 - 0x1F)
     PRIORITY = 0x10  # u8: Message priority (0-255, higher = more urgent)
@@ -382,6 +393,88 @@ class VersionEntry:
     value: bytes
 
 
+@dataclass
+class PutResult:
+    """Result of a successful KV put.
+
+    The ``version`` field is the new version assigned by the server, suitable
+    for CAS on the next write via :class:`PutOptions.cas_version`.
+    """
+
+    version: int
+
+
+@dataclass
+class KVBeginResult:
+    """Result of a successful KV transaction begin.
+
+    ``txn_id`` is the server-assigned transaction handle. ``pinned_hash`` is
+    the partition hash this transaction is bound to — every key written or
+    read inside the transaction must hash to the same partition.
+    """
+
+    txn_id: int
+    pinned_hash: int
+
+
+@dataclass
+class KVCommitResult:
+    """Result of a successful KV transaction commit.
+
+    ``commit_index`` is the Raft log index of the committed batch and
+    ``op_count`` is the number of buffered operations applied atomically.
+    """
+
+    commit_index: int
+    op_count: int
+
+
+@dataclass
+class GetResult:
+    """Result of a KV get that found a key.
+
+    ``kv.get`` returns ``None`` when the key is missing; check for ``None``
+    before dereferencing.
+    """
+
+    value: bytes
+    version: int
+
+
+T = TypeVar("T")
+
+
+@dataclass
+class GetJsonResult(Generic[T]):
+    """Result of a :meth:`KV.json_get` that found the key and path.
+
+    Unlike :class:`GetResult`, ``value`` is the already-parsed JSON (the
+    ``KV_JSON_GET`` wire body is always a JSON document, so it is decoded for
+    you). ``version`` is the document's current version. ``kv.json_get``
+    returns ``None`` when the key or path is missing.
+
+    The value type defaults to ``Any``; the SDK does not validate the shape at
+    runtime.
+    """
+
+    value: Any
+    version: int
+
+
+@dataclass
+class MGetEntry:
+    """One entry in a :meth:`KV.mget` response.
+
+    ``found`` is ``False`` when the key did not exist; in that case ``value``
+    is ``b''`` and ``version`` is ``0``.
+    """
+
+    key: str
+    value: bytes
+    version: int
+    found: bool
+
+
 @dataclass
 class Message:
     """Queue message."""
@@ -440,6 +533,7 @@ class StreamRecord:
 
     id: StreamID = None  # type: ignore[assignment]
     tier: StorageTier = StorageTier.HOT
+    stream: str = ""
     payload: bytes = b""
     headers: dict[str, str] | None = None
 
@@ -458,6 +552,35 @@ class StreamReadResult:
     records: list[StreamRecord]
 
 
+@dataclass
+class PendingEntry:
+    """One entry in a consumer group's Pending Entry List (PEL).
+
+    A delivered-but-unacked message. ``delivery_count`` is how many times the
+    entry has been delivered; ``consumer`` is the consumer that currently owns
+    it. (FLO-102)
+    """
+
+    id: StreamID = None  # type: ignore[assignment]
+    consumer: str = ""
+    delivery_count: int = 0
+
+
+@dataclass
+class StreamClaimResult:
+    """Result of a :meth:`StreamOperations.group_claim` cursor page (FLO-102).
+
+    ``records`` carry payload + headers (same shape as ``group_read``).
+    ``next_cursor`` is the ``start_id`` to pass on the next ``group_claim``
+    call; when ``done`` is ``True`` the PEL has been fully scanned and the loop
+    should stop.
+    """
+
+    records: list[StreamRecord]
+    next_cursor: StreamID = None  # type: ignore[assignment]
+    done: bool = False
+
+
 @dataclass
 class StreamInfo:
     """Stream metadata."""
@@ -498,6 +621,7 @@ class DeleteOptions:
     """Options for KV delete operations."""
 
     namespace: str | None = None
+    if_match: int | None = None  # CAS: only delete when current version equals if_match
 
 
 @dataclass
@@ -518,6 +642,43 @@ class HistoryOptions:
     limit: int | None = None
 
 
+@dataclass
+class KVIncrOptions:
+    """Options for KV incr operations."""
+
+    namespace: str | None = None
+    delta: int | None = None  # default +1 when None
+
+
+@dataclass
+class KVTouchOptions:
+    """Options for KV touch / persist operations."""
+
+    namespace: str | None = None
+    if_match: int | None = None  # CAS: only succeed when current version equals if_match
+
+
+@dataclass
+class KVExistsOptions:
+    """Options for KV exists operations."""
+
+    namespace: str | None = None
+
+
+@dataclass
+class KVJsonOptions:
+    """Options for KV JSON.* operations."""
+
+    namespace: str | None = None
+
+
+@dataclass
+class KVMGetOptions:
+    """Options for KV mget operations."""
+
+    namespace: str | None = None
+
+
 @dataclass
 class EnqueueOptions:
     """Options for queue enqueue operations."""
@@ -769,6 +930,8 @@ class TaskAssignment:
     payload: bytes
     created_at: int
     attempt: int
+    caller_run_id: str = ""
+    caller_workflow_name: str = ""
 
 
 # Alias for backwards compatibility

flo/wire.py

@@ -30,6 +30,7 @@ from .types import (
     Message,
     OpCode,
     OptionTag,
+    PendingEntry,
     ScanResult,
     StatusCode,
     StorageTier,
@@ -597,7 +598,8 @@ def parse_stream_read_response(data: bytes) -> StreamReadResult:
         key_present = data[pos]
         pos += 1
 
-        # Skip key if present
+        # Read key (stream name) if present
+        stream_name = ""
         if key_present != 0:
             if pos + 4 > len(data):
                 raise IncompleteResponseError("Stream record: missing key length")
@@ -605,6 +607,7 @@ def parse_stream_read_response(data: bytes) -> StreamReadResult:
             pos += 4
             if pos + key_len > len(data):
                 raise IncompleteResponseError("Stream record: missing key data")
+            stream_name = data[pos : pos + key_len].decode("utf-8")
             pos += key_len
 
         # Read payload
@@ -618,17 +621,41 @@ def parse_stream_read_response(data: bytes) -> StreamReadResult:
         payload = data[pos : pos + payload_len]
         pos += payload_len
 
-        # Skip header_count (TODO: parse headers)
+        # Read headers: [header_count:u32]([key_len:u32][key][val_len:u32][val])*
         if pos + 4 > len(data):
             raise IncompleteResponseError("Stream record: missing header count")
+        header_count = struct.unpack("<I", data[pos : pos + 4])[0]
         pos += 4
 
+        headers = None
+        if header_count > 0:
+            headers = {}
+            for _ in range(header_count):
+                if pos + 4 > len(data):
+                    raise IncompleteResponseError("Stream record: missing header key length")
+                hkey_len = struct.unpack("<I", data[pos : pos + 4])[0]
+                pos += 4
+                if pos + hkey_len > len(data):
+                    raise IncompleteResponseError("Stream record: missing header key")
+                hkey = data[pos : pos + hkey_len].decode("utf-8")
+                pos += hkey_len
+                if pos + 4 > len(data):
+                    raise IncompleteResponseError("Stream record: missing header value length")
+                hval_len = struct.unpack("<I", data[pos : pos + 4])[0]
+                pos += 4
+                if pos + hval_len > len(data):
+                    raise IncompleteResponseError("Stream record: missing header value")
+                hval = data[pos : pos + hval_len].decode("utf-8")
+                pos += hval_len
+                headers[hkey] = hval
+
         records.append(
             StreamRecord(
                 id=StreamID(timestamp_ms=timestamp_ms, sequence=sequence),
                 tier=tier,
+                stream=stream_name,
                 payload=payload,
-                headers=None,
+                headers=headers,
             )
         )
 
@@ -661,6 +688,33 @@ def parse_stream_info_response(data: bytes) -> StreamInfo:
     )
 
 
+def build_stream_batch_value(
+    payload: bytes,
+    headers: dict[str, str] | None = None,
+) -> bytes:
+    """Frame a single record for stream_append (batch of one).
+
+    Wire format:
+    [record_count:u32=1][payload_len:u32][payload]
+    [header_count:u16]([key_len:u16][key][val_len:u16][val])*
+    """
+    hdrs = headers or {}
+    parts: list[bytes] = [
+        struct.pack("<I", 1),
+        struct.pack("<I", len(payload)),
+        payload,
+        struct.pack("<H", len(hdrs)),
+    ]
+    for key, val in hdrs.items():
+        key_bytes = key.encode("utf-8")
+        val_bytes = val.encode("utf-8")
+        parts.append(struct.pack("<H", len(key_bytes)))
+        parts.append(key_bytes)
+        parts.append(struct.pack("<H", len(val_bytes)))
+        parts.append(val_bytes)
+    return b"".join(parts)
+
+
 def serialize_group_value(group: str, consumer: str) -> bytes:
     """Serialize group and consumer names for consumer group operations.
 
@@ -697,6 +751,92 @@ def serialize_group_ack_value(group: str, ids: list[StreamID], consumer: str = "
     return bytes(result)
 
 
+def serialize_group_pending_value(group: str, consumer: str = "") -> bytes:
+    """Serialize group (and optional consumer filter) for STREAM_GROUP_PENDING.
+
+    Format: [group_len:u16][group]([consumer_len:u16][consumer])?
+
+    An empty ``consumer`` returns the whole group's PEL; a non-empty one filters
+    to that consumer's entries.
+    """
+    group_bytes = group.encode("utf-8")
+
+    result = bytearray()
+    result.extend(struct.pack("<H", len(group_bytes)))
+    result.extend(group_bytes)
+    if consumer:
+        consumer_bytes = consumer.encode("utf-8")
+        result.extend(struct.pack("<H", len(consumer_bytes)))
+        result.extend(consumer_bytes)
+    return bytes(result)
+
+
+def serialize_group_claim_value(
+    group: str,
+    consumer: str,
+    min_idle_ms: int,
+    start_id: StreamID,
+    count: int,
+) -> bytes:
+    """Serialize the request value for STREAM_GROUP_CLAIM (FLO-102).
+
+    Format: [group_len:u16][group][consumer_len:u16][consumer]
+            [min_idle_ms:u32][start_ts:u64][start_seq:u64][count:u32]
+    """
+    group_bytes = group.encode("utf-8")
+    consumer_bytes = consumer.encode("utf-8")
+
+    result = bytearray()
+    result.extend(struct.pack("<H", len(group_bytes)))
+    result.extend(group_bytes)
+    result.extend(struct.pack("<H", len(consumer_bytes)))
+    result.extend(consumer_bytes)
+    result.extend(struct.pack("<I", min_idle_ms))
+    result.extend(struct.pack("<Q", start_id.timestamp_ms))
+    result.extend(struct.pack("<Q", start_id.sequence))
+    result.extend(struct.pack("<I", count))
+    return bytes(result)
+
+
+def parse_pending_entries(data: bytes) -> list[PendingEntry]:
+    """Parse the STREAM_GROUP_PENDING response (FLO-102).
+
+    Format: [count:u32]([ts:u64][seq:u64][delivery_count:u32]
+            [consumer_len:u16][consumer])*
+    """
+    if len(data) < 4:
+        return []
+
+    pos = 0
+    count = struct.unpack("<I", data[pos : pos + 4])[0]
+    pos += 4
+
+    entries: list[PendingEntry] = []
+    for _ in range(count):
+        if pos + 8 + 8 + 4 + 2 > len(data):
+            raise IncompleteResponseError("Pending entry: truncated header")
+        ts = struct.unpack("<Q", data[pos : pos + 8])[0]
+        pos += 8
+        seq = struct.unpack("<Q", data[pos : pos + 8])[0]
+        pos += 8
+        delivery_count = struct.unpack("<I", data[pos : pos + 4])[0]
+        pos += 4
+        clen = struct.unpack("<H", data[pos : pos + 2])[0]
+        pos += 2
+        if pos + clen > len(data):
+            raise IncompleteResponseError("Pending entry: truncated consumer")
+        consumer = data[pos : pos + clen].decode("utf-8")
+        pos += clen
+        entries.append(
+            PendingEntry(
+                id=StreamID(timestamp_ms=ts, sequence=seq),
+                consumer=consumer,
+                delivery_count=delivery_count,
+            )
+        )
+    return entries
+
+
 # =============================================================================
 # Action/Worker Value Serialization
 # =============================================================================
@@ -760,7 +900,8 @@ def serialize_action_invoke_value(
     """Serialize action invoke value.
 
     Format: [priority:u8][delay_ms:i64][has_caller:u8]
-            [has_idempotency_key:u8][key_len:u16]?[key]?[input...]
+            [has_idempotency_key:u8][key_len:u16]?[key]?
+            [has_labels:u8][labels_len:u16]?[labels]?[input...]
     """
     result = bytearray()
 
@@ -782,6 +923,9 @@ def serialize_action_invoke_value(
     else:
         result.append(0)
 
+    # labels (none)
+    result.append(0)
+
     # input
     result.extend(input_data)
 
@@ -945,7 +1089,9 @@ def parse_task_assignment(data: bytes) -> "types.TaskAssignment":
     """Parse task assignment from server response.
 
     Format: [task_id_len:u16][task_id][task_type_len:u16][task_type]
-            [created_at:i64][attempt:u32][payload...]
+            [created_at:i64][attempt:u32]
+            [has_caller:u8]([run_id_len:u16][run_id][wf_name_len:u16][wf_name])?
+            [payload...]
     """
     if len(data) < 10:
         raise ValueError("Incomplete task assignment response")
@@ -982,6 +1128,32 @@ def parse_task_assignment(data: bytes) -> "types.TaskAssignment":
     attempt = struct.unpack_from("<I", data, pos)[0]
     pos += 4
 
+    # caller block: [has_caller:u8][run_id_len:u16][run_id][wf_name_len:u16][wf_name]
+    if pos + 1 > len(data):
+        raise ValueError("Incomplete task assignment: missing has_caller")
+    has_caller = data[pos]
+    pos += 1
+    caller_run_id = ""
+    caller_workflow_name = ""
+    if has_caller == 1:
+        if pos + 2 > len(data):
+            raise ValueError("Incomplete task assignment: missing caller_run_id length")
+        crid_len = struct.unpack_from("<H", data, pos)[0]
+        pos += 2
+        if pos + crid_len > len(data):
+            raise ValueError("Incomplete task assignment: missing caller_run_id")
+        caller_run_id = data[pos : pos + crid_len].decode("utf-8")
+        pos += crid_len
+
+        if pos + 2 > len(data):
+            raise ValueError("Incomplete task assignment: missing caller_workflow_name length")
+        cwf_len = struct.unpack_from("<H", data, pos)[0]
+        pos += 2
+        if pos + cwf_len > len(data):
+            raise ValueError("Incomplete task assignment: missing caller_workflow_name")
+        caller_workflow_name = data[pos : pos + cwf_len].decode("utf-8")
+        pos += cwf_len
+
     # payload (rest of data)
     payload = data[pos:]
 
@@ -991,4 +1163,6 @@ def parse_task_assignment(data: bytes) -> "types.TaskAssignment":
         payload=payload,
         created_at=created_at,
         attempt=attempt,
+        caller_run_id=caller_run_id,
+        caller_workflow_name=caller_workflow_name,
     )

flo/worker.py

@@ -666,6 +666,15 @@ class StreamWorkerOptions:
     batch_size: int = 10
     block_ms: int = 30000
     message_timeout: float = 300.0  # 5 minutes
+    # Drain this consumer's pending (delivered-but-unacked) entries via
+    # group_claim after a reconnect, before resuming normal group_read.
+    # Required for at-least-once across reconnects: group_read alone only
+    # returns records past last_delivered_id. Set False for at-most-once.
+    redeliver_pending_on_reconnect: bool = True
+    # Minimum idle time (ms) an entry must have before the reconnect drain
+    # claims it. 0 = drain everything this consumer already owns; > 0 also
+    # steals entries abandoned by other (dead) consumers idle at least this long.
+    claim_min_idle_ms: int = 0
 
 
 @dataclass
@@ -749,6 +758,8 @@ class StreamWorker:
         batch_size: int = 10,
         block_ms: int = 30000,
         message_timeout: float = 300.0,
+        redeliver_pending_on_reconnect: bool = True,
+        claim_min_idle_ms: int = 0,
     ):
         self._parent_client = parent_client
         self._handler = handler
@@ -761,6 +772,8 @@ class StreamWorker:
             batch_size=batch_size,
             block_ms=block_ms,
             message_timeout=message_timeout,
+            redeliver_pending_on_reconnect=redeliver_pending_on_reconnect,
+            claim_min_idle_ms=claim_min_idle_ms,
         )
 
         self._client: FloClient | None = None
@@ -805,18 +818,19 @@ class StreamWorker:
         await self._client.connect()
 
         try:
-            # Join consumer group
-            await self._client.stream.group_join(
-                self.config.stream,
-                self.config.group,
-                self.config.consumer,
-            )
-            logger.info(f"Joined consumer group {self.config.group} on stream {self.config.stream}")
-
+            # Semaphore and running state must be set up before the join: a
+            # connection error during the join triggers _handle_reconnect ->
+            # _drain_pending, which acquires the semaphore.
             self._semaphore = asyncio.Semaphore(self.config.concurrency)
             self._running = True
             self._stop_event.clear()
 
+            # Join consumer group. A connection error here (a stalled or dropped
+            # connection at startup) is retried via reconnect rather than
+            # killing the worker — mirroring the poll loop and ack retry.
+            await self._join_group()
+            logger.info(f"Joined consumer group {self.config.group} on stream {self.config.stream}")
+
             await self._poll_loop()
 
         finally:
@@ -900,8 +914,41 @@ class StreamWorker:
                     logger.error(f"Stream read error: {e}, retrying...")
                     await asyncio.sleep(1)
 
+    async def _join_group(self) -> None:
+        """Join the consumer group, retrying on connection errors.
+
+        A connection error (a stalled or dropped connection at startup) triggers
+        a reconnect-and-retry instead of failing the worker. Non-connection
+        errors (e.g. an invalid request) remain fatal and are raised.
+        """
+        assert self._client is not None
+        while True:
+            if self._stop_event.is_set():
+                return
+            try:
+                await self._client.stream.group_join(
+                    self.config.stream,
+                    self.config.group,
+                    self.config.consumer,
+                )
+                return
+            except Exception as e:
+                if not is_connection_error(e):
+                    raise
+                logger.warning(f"Connection lost during group join: {e}, reconnecting...")
+                with contextlib.suppress(Exception):
+                    await self._handle_reconnect()
+                await asyncio.sleep(1)
+
     async def _handle_reconnect(self) -> None:
-        """Reconnect and re-join the consumer group."""
+        """Reconnect, re-join the consumer group, and drain pending entries.
+
+        After re-joining, this consumer's pending (delivered-but-unacked)
+        entries are re-processed via a group_claim cursor loop before normal
+        reads resume — group_read only returns records past last_delivered_id,
+        so without this an in-flight record at crash time is never re-surfaced.
+        See ``redeliver_pending_on_reconnect``.
+        """
         assert self._client is not None
         await self._client.reconnect()
         await self._client.stream.group_join(
@@ -909,6 +956,55 @@ class StreamWorker:
             self.config.group,
             self.config.consumer,
         )
+        if self.config.redeliver_pending_on_reconnect:
+            await self._drain_pending()
+
+    async def _drain_pending(self) -> None:
+        """Re-process this consumer's pending entries via a group_claim loop.
+
+        Records flow through the same _process_record path (and concurrency
+        gate) as live reads, so a handler that succeeds acks and clears the PEL
+        entry. Bounded per stream by a max page count so a pathological backlog
+        can't wedge the reconnect indefinitely; leftovers are reclaimed on the
+        next reconnect or by group_read once acked.
+        """
+        if self._semaphore is None or self._client is None:
+            # start() has not initialized the concurrency gate yet.
+            return
+
+        max_pages = 1000  # safety cap: max_pages * batch_size entries
+        cursor = StreamID()  # MIN — scan from the start of the PEL
+        pages = 0
+        while True:
+            if self._stop_event.is_set():
+                return
+            try:
+                result = await self._client.stream.group_claim(
+                    self.config.stream,
+                    self.config.group,
+                    self.config.consumer,
+                    self.config.claim_min_idle_ms,
+                    cursor,
+                    self.config.batch_size,
+                )
+            except Exception as e:
+                # Don't fail the reconnect on a drain error; normal reads resume.
+                logger.warning(f"Pending drain claim failed: {e}")
+                return
+
+            for record in result.records:
+                await self._semaphore.acquire()  # same gate as live reads
+                if self._stop_event.is_set():
+                    self._semaphore.release()
+                    return
+                task = asyncio.create_task(self._process_record(record))
+                self._tasks.add(task)
+                task.add_done_callback(self._tasks.discard)
+
+            pages += 1
+            if result.done or not result.records or pages >= max_pages:
+                return
+            cursor = result.next_cursor
 
     async def _ack_with_retry(
         self, record_ids: list[StreamID], options: StreamGroupAckOptions

{flo_python-0.1.0.dev4.dist-info → flo_python-0.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: flo-python
-Version: 0.1.0.dev4
+Version: 0.1.1
 Summary: Python SDK for the Flo distributed systems platform
 Project-URL: Homepage, https://github.com/floruntime/flo-python
 Project-URL: Documentation, https://github.com/floruntime/flo-python#readme
@@ -35,7 +35,7 @@ Python SDK for the [Flo](https://github.com/floruntime) distributed systems plat
 ## Installation
 
 ```bash
-pip install flo
+pip install flo-python
 ```
 
 ## Quick Start