flo-python 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl

flo/__init__.py

@@ -34,6 +34,7 @@ Example:
 from .client import FloClient
 from .exceptions import (
     BadRequestError,
+    BlockTooLongError,
     ConflictError,
     ConnectionFailedError,
     FloError,
@@ -52,16 +53,19 @@ from .exceptions import (
     PayloadTooLargeError,
     ProtocolError,
     RateLimitedError,
+    RequestTimeoutError,
     ServerError,
+    TableMismatchError,
     UnauthorizedError,
+    UnavailableError,
     UnexpectedEofError,
-    UnsupportedVersionError,
     ValidationError,
     ValueTooLargeError,
     is_connection_error,
 )
 from .kv_txn import Transaction, TxnFinishedError, TxnUnsupportedOpError
 from .processing import ProcessingOperations
+from .retry import DEFAULT_RETRY_POLICY, RetryPolicy
 from .types import (
     AckOptions,
     ActionDeleteOptions,
@@ -74,6 +78,7 @@ from .types import (
     ActionRunStatus,
     ActionStatusOptions,
     ActionType,
+    Condition,
     DeleteOptions,
     DequeueOptions,
     DequeueResult,
@@ -84,14 +89,14 @@ from .types import (
     GetOptions,
     GetResult,
     HistoryOptions,
+    IncrResult,
     KVBeginResult,
     KVCommitResult,
-    KVEntry,
     KVExistsOptions,
+    KVExpireOptions,
     KVIncrOptions,
     KVJsonOptions,
     KVMGetOptions,
-    KVTouchOptions,
     Message,
     MGetEntry,
     NackOptions,
@@ -102,9 +107,6 @@ from .types import (
     ProcessingCancelOptions,
     ProcessingListEntry,
     ProcessingListOptions,
-    ProcessingRescaleOptions,
-    ProcessingRestoreOptions,
-    ProcessingSavepointOptions,
     ProcessingStatusOptions,
     ProcessingStatusResult,
     ProcessingStopOptions,
@@ -113,6 +115,9 @@ from .types import (
     ProcessingSyncResult,
     PutOptions,
     PutResult,
+    Ran,
+    Reason,
+    RunStatus,
     ScanOptions,
     ScanResult,
     StatusCode,
@@ -131,9 +136,10 @@ from .types import (
     StreamReadResult,
     StreamRecord,
     StreamTrimOptions,
+    StreamTrimResult,
     TaskAssignment,
-    TouchOptions,
     VersionEntry,
+    WaitFor,
     WorkerAwaitOptions,
     WorkerAwaitResult,
     WorkerCompleteOptions,
@@ -142,8 +148,9 @@ from .types import (
     WorkerListOptions,
     WorkerListResult,
     WorkerRegisterOptions,
-    WorkerTask,
+    WorkerStatus,
     WorkerTouchOptions,
+    WorkerType,
     WorkflowCancelOptions,
     WorkflowCreateOptions,
     WorkflowDisableOptions,
@@ -170,7 +177,7 @@ from .worker import (
 )
 from .workflows import WorkflowOperations
 
-__version__ = "0.1.1"
+__version__ = "0.1.2"
 
 __all__ = [
     "Transaction",
@@ -182,13 +189,16 @@ __all__ = [
     "KVIncrOptions",
     "KVJsonOptions",
     "KVMGetOptions",
-    "KVTouchOptions",
+    "KVExpireOptions",
     "GetResult",
     "GetJsonResult",
     "MGetEntry",
     "PutResult",
+    "IncrResult",
     # Client
     "FloClient",
+    "RetryPolicy",
+    "DEFAULT_RETRY_POLICY",
     # High-level Worker API
     "ActionWorker",
     "ActionWorkerOptions",
@@ -207,9 +217,10 @@ __all__ = [
     "ConnectionFailedError",
     "InvalidEndpointError",
     "UnexpectedEofError",
+    "RequestTimeoutError",
     "ProtocolError",
     "InvalidMagicError",
-    "UnsupportedVersionError",
+    "TableMismatchError",
     "InvalidChecksumError",
     "PayloadTooLargeError",
     "IncompleteResponseError",
@@ -217,28 +228,33 @@ __all__ = [
     "NamespaceTooLargeError",
     "KeyTooLargeError",
     "ValueTooLargeError",
+    "BlockTooLongError",
     "ServerError",
     "NotFoundError",
     "BadRequestError",
     "ConflictError",
     "UnauthorizedError",
     "OverloadedError",
+    "UnavailableError",
     "RateLimitedError",
     "InternalServerError",
     "GenericServerError",
     "is_connection_error",
     # Types
     "OpCode",
+    "Ran",
+    "Reason",
     "StatusCode",
     "OptionTag",
     # Result types
-    "KVEntry",
     "ScanResult",
     "VersionEntry",
     "Message",
     "DequeueResult",
     # Options
+    "Condition",
     "GetOptions",
+    "WaitFor",
     "PutOptions",
     "DeleteOptions",
     "ScanOptions",
@@ -250,7 +266,6 @@ __all__ = [
     "DlqListOptions",
     "DlqRequeueOptions",
     "PeekOptions",
-    "TouchOptions",
     # Stream types
     "StreamID",
     "StorageTier",
@@ -264,6 +279,7 @@ __all__ = [
     "StreamAppendOptions",
     "StreamReadOptions",
     "StreamTrimOptions",
+    "StreamTrimResult",
     "StreamInfoOptions",
     "StreamGroupJoinOptions",
     "StreamGroupReadOptions",
@@ -271,6 +287,7 @@ __all__ = [
     "StreamGroupNackOptions",
     # Action types
     "ActionType",
+    "RunStatus",
     "ActionInfo",
     "ActionRunStatus",
     "ActionInvokeResult",
@@ -283,10 +300,11 @@ __all__ = [
     "ActionDeleteOptions",
     # Worker types
     "TaskAssignment",
-    "WorkerTask",
     "WorkerAwaitResult",
     "WorkerInfo",
     "WorkerListResult",
+    "WorkerStatus",
+    "WorkerType",
     # Worker options
     "WorkerRegisterOptions",
     "WorkerAwaitOptions",
@@ -316,9 +334,6 @@ __all__ = [
     "ProcessingListOptions",
     "ProcessingStopOptions",
     "ProcessingCancelOptions",
-    "ProcessingSavepointOptions",
-    "ProcessingRestoreOptions",
-    "ProcessingRescaleOptions",
     "ProcessingSyncOptions",
     "ProcessingStatusResult",
     "ProcessingListEntry",

flo/actions.py

@@ -9,6 +9,12 @@ For a higher-level API, see `Worker` in worker.py.
 
 from typing import TYPE_CHECKING
 
+from .kv_body import (
+    decode_action_invoke_answer,
+    decode_action_status_answer,
+    decode_worker_list_answer,
+    encode_worker_list_body,
+)
 from .types import (
     ActionDeleteOptions,
     ActionInvokeOptions,
@@ -39,7 +45,6 @@ from .wire import (
     serialize_worker_await_value,
     serialize_worker_complete_value,
     serialize_worker_fail_value,
-    serialize_worker_list_value,
     serialize_worker_register_value,
     serialize_worker_touch_value,
 )
@@ -78,7 +83,7 @@ class ActionOperations:
             action_type=action_type,
             timeout_ms=opts.timeout_ms,
             max_retries=opts.max_retries,
-            description=opts.description,
+            owner=opts.owner,
         )
 
         await self._client._send_and_check(
@@ -109,8 +114,7 @@ class ActionOperations:
 
         value = serialize_action_invoke_value(
             input_data=input_data,
-            priority=opts.priority,
-            idempotency_key=opts.idempotency_key,
+            labels=opts.labels,
         )
 
         response = await self._client._send_and_check(
@@ -120,9 +124,7 @@ class ActionOperations:
             value,
         )
 
-        # Response contains run_id as string
-        run_id = response.data.decode("utf-8") if response.data else ""
-        return ActionInvokeResult(run_id=run_id)
+        return ActionInvokeResult(run_id=decode_action_invoke_answer(response.data))
 
     async def status(
         self,
@@ -148,13 +150,7 @@ class ActionOperations:
             b"",
         )
 
-        # Parse status response - format TBD based on server implementation
-        # For now return basic status
-        return ActionRunStatus(
-            run_id=run_id,
-            status="unknown",
-            result=response.data if response.data else None,
-        )
+        return decode_action_status_answer(run_id, response.data)
 
     async def list(
         self,
@@ -280,7 +276,7 @@ class WorkerOperations:
         Args:
             worker_id: Worker identifier.
             task_types: Task types to listen for.
-            options: Optional await options (block_ms, timeout_ms).
+            options: Optional await options (block_ms).
 
         Returns:
             WorkerAwaitResult with task if available.
@@ -290,12 +286,9 @@ class WorkerOperations:
 
         value = serialize_worker_await_value(task_types)
 
-        # Build options for block_ms and timeout_ms
         options_builder = OptionsBuilder()
         if opts.block_ms is not None:
             options_builder.add_u32(OptionTag.BLOCK_MS, opts.block_ms)
-        if opts.timeout_ms is not None:
-            options_builder.add_u32(OptionTag.TIMEOUT_MS, opts.timeout_ms)
 
         response = await self._client._send_and_check(
             OpCode.ACTION_AWAIT,
@@ -402,25 +395,25 @@ class WorkerOperations:
         self,
         options: WorkerListOptions | None = None,
     ) -> WorkerListResult:
-        """List registered workers.
+        """List a page of the namespace's workers, in id order.
 
         Args:
-            options: Optional list options.
+            options: The page's limit and cursor (the last page's ``cursor``;
+                None starts).
 
         Returns:
-            WorkerListResult with list of workers.
+            WorkerListResult with the page's workers and the next page's
+            cursor, None on the last page.
         """
         opts = options or WorkerListOptions()
         namespace = self._client.get_namespace(opts.namespace)
 
-        value = serialize_worker_list_value(opts.limit)
-
-        await self._client._send_and_check(
+        response = await self._client._send_and_check(
             OpCode.WORKER_LIST,
             namespace,
             b"",
-            value,
+            encode_worker_list_body(opts.limit, opts.cursor),
         )
 
-        # Parse list response - format TBD based on server implementation
-        return WorkerListResult(workers=[])
+        workers, cursor = decode_worker_list_answer(response.data)
+        return WorkerListResult(workers=workers, cursor=cursor)

flo/client.py

@@ -10,19 +10,31 @@ from typing import TYPE_CHECKING
 
 from .exceptions import (
     ConnectionFailedError,
+    InvalidChecksumError,
     InvalidEndpointError,
     NotConnectedError,
+    ProtocolError,
+    RequestTimeoutError,
     UnexpectedEofError,
     raise_for_status,
 )
-from .types import HEADER_SIZE, OpCode, StatusCode
-from .wire import RawResponse, parse_response_header, serialize_request
+from .retry import DEFAULT_RETRY_POLICY, RetryPolicy, _policy_backoff, _refused_unrun
+from .types import _DEFAULT_WORKER_BLOCK_MS, HEADER_SIZE, OpCode, OptionTag, StatusCode
+from .wire import (
+    OptionsIterator,
+    RawResponse,
+    compute_crc32,
+    parse_response_header,
+    serialize_request,
+)
 
 if TYPE_CHECKING:
     from .worker import ActionWorker, StreamRecordHandler, StreamWorker
 
 logger = logging.getLogger("flo")
 
+_SERVER_AWAIT_DEFAULT_MS = 30_000
+
 
 class FloClient:
     """Async client for Flo server.
@@ -40,19 +52,28 @@ class FloClient:
         namespace: str = "default",
         timeout_ms: int = 5000,
         debug: bool = False,
+        retry: RetryPolicy = DEFAULT_RETRY_POLICY,
     ):
         """Initialize Flo client.
 
         Args:
             endpoint: Server endpoint in "host:port" format.
             namespace: Default namespace for operations.
-            timeout_ms: Connection and operation timeout in milliseconds.
+            timeout_ms: Connection and operation timeout in milliseconds. A
+                blocking call (block_ms) is allowed this long on
+                top of its wait. A call that times out or is cancelled drops
+                the connection; call reconnect() before reusing the client.
             debug: Enable debug logging.
+            retry: How a request the server refused without running it
+                (overloaded or unavailable, ran = no) is sent again; backoffs
+                are in milliseconds. RetryPolicy(max_attempts=1) turns
+                retrying off. Workers made from this client use it too.
         """
         self._endpoint = endpoint
         self._namespace = namespace
         self._timeout = timeout_ms / 1000.0  # Convert to seconds
         self._debug = debug
+        self._retry = retry
 
         self._reader: asyncio.StreamReader | None = None
         self._writer: asyncio.StreamWriter | None = None
@@ -187,13 +208,16 @@ class FloClient:
             timeout: Maximum total time to keep retrying (default 5 min).
         """
         async with self._lock:
-            # Close existing connection
-            if self._writer:
-                self._writer.close()
-                with contextlib.suppress(Exception):
-                    await self._writer.wait_closed()
+            # Close existing connection. It is detached before the await so a
+            # concurrent reconnect that installs its connection meanwhile
+            # keeps it.
+            old = self._writer
+            if old:
                 self._writer = None
                 self._reader = None
+                old.close()
+                with contextlib.suppress(Exception):
+                    await old.wait_closed()
 
         backoff = 1.0
         max_backoff = 30.0
@@ -203,10 +227,16 @@ class FloClient:
         while True:
             attempt += 1
             try:
-                self._reader, self._writer = await asyncio.wait_for(
+                reader, writer = await asyncio.wait_for(
                     asyncio.open_connection(self._host, self._port),
                     timeout=self._timeout,
                 )
+                # A concurrent reconnect may have installed its connection
+                # first; close it rather than leak it.
+                replaced = self._writer
+                self._reader, self._writer = reader, writer
+                if replaced is not None:
+                    replaced.close()
                 if self._debug:
                     logger.debug(f"[flo] Reconnected to {self._endpoint} (attempt {attempt})")
                 return
@@ -244,6 +274,24 @@ class FloClient:
         self._request_id += 1
         return self._request_id
 
+    @staticmethod
+    def _server_wait_ms(op_code: OpCode, value: bytes, options: bytes) -> int:
+        """How long the server may hold this request before answering."""
+        if op_code == OpCode.KV_GET:
+            # kv.get's body opens with wait_ms: [present:u8][u64].
+            if len(value) >= 9 and value[0] == 1:
+                return int.from_bytes(value[1:9], "little")
+            return 0
+        waits = {
+            opt.tag: opt.as_u32() or 0
+            for opt in OptionsIterator(options)
+            if opt.tag == OptionTag.BLOCK_MS
+        }
+        if op_code == OpCode.ACTION_AWAIT:
+            # An await that names no block_ms waits the server's default.
+            waits.setdefault(OptionTag.BLOCK_MS, _SERVER_AWAIT_DEFAULT_MS)
+        return max(waits.values(), default=0)
+
     async def _send_request(
         self,
         op_code: OpCode,
@@ -252,7 +300,35 @@ class FloClient:
         value: bytes,
         options: bytes = b"",
     ) -> RawResponse:
-        """Send a request and wait for response.
+        """Send a request and wait for its response, sending it again as the
+        retry policy allows while the server refuses it without running it.
+
+        Each try is a new request with a new id. The wait between tries is
+        taken outside the connection lock, so other calls proceed meanwhile.
+        """
+        wait = _policy_backoff(self._retry)
+        attempt = 1
+        while True:
+            response = await self._send_once(op_code, namespace, key, value, options)
+            if attempt >= self._retry.max_attempts or not _refused_unrun(
+                response.status, response.data
+            ):
+                return response
+            d = wait.next()
+            if self._debug:
+                logger.debug(f"[flo] {response.status.name}, not run: retrying in {d:.3f}s")
+            await asyncio.sleep(d)
+            attempt += 1
+
+    async def _send_once(
+        self,
+        op_code: OpCode,
+        namespace: str,
+        key: bytes,
+        value: bytes,
+        options: bytes = b"",
+    ) -> RawResponse:
+        """Send a request, under a new request id, and wait for its response.
 
         Args:
             op_code: Operation code.
@@ -272,6 +348,10 @@ class FloClient:
             raise NotConnectedError("Not connected to server")
 
         async with self._lock:
+            # A call that held the lock before this one may have dropped the
+            # connection.
+            if self._writer is None or self._reader is None:
+                raise NotConnectedError("Not connected to server")
             request_id = self._next_request_id()
 
             # Serialize request
@@ -287,46 +367,74 @@ class FloClient:
             if self._debug:
                 logger.debug(f"[flo] -> {op_code.name} ns={namespace} key={key!r}")
 
-            # Send request
-            self._writer.write(request)
-            await self._writer.drain()
-
-            # Read response header
+            # The server may hold a blocking request for its whole wait, so it
+            # gets the operation timeout on top of that wait.
+            read_timeout = self._timeout + self._server_wait_ms(op_code, value, options) / 1000.0
+            writer, reader = self._writer, self._reader
             try:
-                header_data = await asyncio.wait_for(
-                    self._reader.readexactly(HEADER_SIZE),
-                    timeout=self._timeout,
-                )
-            except asyncio.IncompleteReadError as e:
-                raise UnexpectedEofError("Connection closed while reading response header") from e
-
-            # Parse header to get data length
-            status, data_len, resp_request_id, crc = parse_response_header(header_data)
+                writer.write(request)
+                await writer.drain()
 
-            # Read response data
-            if data_len > 0:
                 try:
-                    response_data = await asyncio.wait_for(
-                        self._reader.readexactly(data_len),
-                        timeout=self._timeout,
+                    header_data = await asyncio.wait_for(
+                        reader.readexactly(HEADER_SIZE),
+                        timeout=read_timeout,
                     )
                 except asyncio.IncompleteReadError as e:
-                    raise UnexpectedEofError("Connection closed while reading response data") from e
-            else:
-                response_data = b""
+                    raise UnexpectedEofError(
+                        "Connection closed while reading response header"
+                    ) from e
+                except asyncio.TimeoutError as e:
+                    raise RequestTimeoutError(f"No response to {op_code.name}") from e
+
+                status, data_len, resp_request_id, crc = parse_response_header(header_data)
+
+                if data_len > 0:
+                    try:
+                        response_data = await asyncio.wait_for(
+                            reader.readexactly(data_len),
+                            timeout=self._timeout,
+                        )
+                    except asyncio.IncompleteReadError as e:
+                        raise UnexpectedEofError(
+                            "Connection closed while reading response data"
+                        ) from e
+                    except asyncio.TimeoutError as e:
+                        raise RequestTimeoutError(
+                            f"Response to {op_code.name} stalled mid-frame"
+                        ) from e
+                else:
+                    response_data = b""
+
+                computed_crc = compute_crc32(header_data, response_data)
+                if computed_crc != crc:
+                    raise InvalidChecksumError(
+                        f"CRC32 mismatch: 0x{computed_crc:08X} != 0x{crc:08X}"
+                    )
+                if resp_request_id == 0:
+                    # The server answers a request it could not parse with
+                    # id 0 and then closes; surface its error.
+                    raise_for_status(status, response_data)
+                if resp_request_id != request_id:
+                    raise ProtocolError(
+                        f"Response is for request {resp_request_id}, expected {request_id}"
+                    )
+            except BaseException:
+                # The reply to an abandoned request (timed out, cancelled, cut
+                # off mid-frame) may still arrive and would be read as the
+                # answer to the next one, and after a corrupt or mismatched
+                # frame the stream position is unknown, so the connection is
+                # dropped; the caller reconnects.
+                writer.close()
+                # reconnect() swaps the connection outside the lock.
+                if self._writer is writer:
+                    self._writer = None
+                    self._reader = None
+                raise
 
             if self._debug:
                 logger.debug(f"[flo] <- {status.name} {len(response_data)} bytes")
 
-            # Verify CRC32
-            from .wire import compute_crc32
-
-            computed_crc = compute_crc32(header_data, response_data)
-            if computed_crc != crc:
-                from .exceptions import InvalidChecksumError
-
-                raise InvalidChecksumError(f"CRC32 mismatch: 0x{computed_crc:08X} != 0x{crc:08X}")
-
             return RawResponse(status=status, data=response_data, request_id=resp_request_id)
 
     async def _send_and_check(
@@ -376,7 +484,8 @@ class FloClient:
         worker_id: str | None = None,
         concurrency: int = 10,
         action_timeout: float = 300.0,
-        block_ms: int = 30000,
+        block_ms: int = _DEFAULT_WORKER_BLOCK_MS,
+        auto_touch: bool = False,
     ) -> "ActionWorker":
         """Create a new ActionWorker from this client.
 
@@ -387,7 +496,11 @@ class FloClient:
             worker_id: Unique worker identifier (auto-generated if not provided).
             concurrency: Maximum number of concurrent actions.
             action_timeout: Timeout for action handlers in seconds.
-            block_ms: Timeout for blocking dequeue in milliseconds.
+            block_ms: Long-poll wait per await in milliseconds, at most 300000.
+                0 means the 30000 default.
+            auto_touch: Touch each run every third of its action's timeout
+                while its handler runs. Off by default: a handler that runs
+                past the timeout without touching is taken back and run again.
 
         Returns:
             A new ActionWorker instance ready to register actions and start.
@@ -408,6 +521,7 @@ class FloClient:
             concurrency=concurrency,
             action_timeout=action_timeout,
             block_ms=block_ms,
+            auto_touch=auto_touch,
         )
 
     def new_stream_worker(
@@ -420,7 +534,7 @@ class FloClient:
         worker_id: str | None = None,
         concurrency: int = 10,
         batch_size: int = 10,
-        block_ms: int = 30000,
+        block_ms: int = _DEFAULT_WORKER_BLOCK_MS,
         message_timeout: float = 300.0,
         redeliver_pending_on_reconnect: bool = True,
         claim_min_idle_ms: int = 0,
@@ -439,7 +553,8 @@ class FloClient:
             worker_id: Unique worker identifier (auto-generated if not provided).
             concurrency: Maximum number of concurrent record handlers.
             batch_size: Number of records to fetch per poll.
-            block_ms: Timeout for blocking read in milliseconds.
+            block_ms: Long-poll wait per read in milliseconds, at most 300000.
+                0 means the 30000 default.
             message_timeout: Timeout for record handlers in seconds.
             redeliver_pending_on_reconnect: Drain this consumer's pending
                 (delivered-but-unacked) entries via group_claim after a