flo-python 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

flo/__init__.py

@@ -49,12 +49,14 @@ from .exceptions import (
     NonRetryableError,
     NotConnectedError,
     NotFoundError,
+    OutcomePendingError,
     OverloadedError,
     PayloadTooLargeError,
     ProtocolError,
     RateLimitedError,
     RequestTimeoutError,
     ServerError,
+    StaleNotFoundError,
     TableMismatchError,
     UnauthorizedError,
     UnavailableError,
@@ -85,10 +87,12 @@ from .types import (
     DlqListOptions,
     DlqRequeueOptions,
     EnqueueOptions,
+    ExistsResult,
     GetJsonResult,
     GetOptions,
     GetResult,
     HistoryOptions,
+    HistoryResult,
     IncrResult,
     KVBeginResult,
     KVCommitResult,
@@ -113,6 +117,7 @@ from .types import (
     ProcessingSubmitOptions,
     ProcessingSyncOptions,
     ProcessingSyncResult,
+    ProcessKind,
     PutOptions,
     PutResult,
     Ran,
@@ -143,10 +148,12 @@ from .types import (
     WorkerAwaitOptions,
     WorkerAwaitResult,
     WorkerCompleteOptions,
+    WorkerDetails,
     WorkerFailOptions,
     WorkerInfo,
     WorkerListOptions,
     WorkerListResult,
+    WorkerProcess,
     WorkerRegisterOptions,
     WorkerStatus,
     WorkerTouchOptions,
@@ -177,7 +184,7 @@ from .worker import (
 )
 from .workflows import WorkflowOperations
 
-__version__ = "0.1.2"
+__version__ = "0.1.3"
 
 __all__ = [
     "Transaction",
@@ -191,6 +198,8 @@ __all__ = [
     "KVMGetOptions",
     "KVExpireOptions",
     "GetResult",
+    "HistoryResult",
+    "ExistsResult",
     "GetJsonResult",
     "MGetEntry",
     "PutResult",
@@ -235,6 +244,8 @@ __all__ = [
     "ConflictError",
     "UnauthorizedError",
     "OverloadedError",
+    "OutcomePendingError",
+    "StaleNotFoundError",
     "UnavailableError",
     "RateLimitedError",
     "InternalServerError",
@@ -301,7 +312,10 @@ __all__ = [
     # Worker types
     "TaskAssignment",
     "WorkerAwaitResult",
+    "ProcessKind",
+    "WorkerDetails",
     "WorkerInfo",
+    "WorkerProcess",
     "WorkerListResult",
     "WorkerStatus",
     "WorkerType",

flo/actions.py

@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
 from .kv_body import (
     decode_action_invoke_answer,
     decode_action_status_answer,
+    decode_worker_info_answer,
     decode_worker_list_answer,
     encode_worker_list_body,
 )
@@ -30,10 +31,12 @@ from .types import (
     WorkerAwaitOptions,
     WorkerAwaitResult,
     WorkerCompleteOptions,
+    WorkerDetails,
     WorkerFailOptions,
     WorkerListOptions,
     WorkerListResult,
     WorkerRegisterOptions,
+    WorkerStatus,
     WorkerTouchOptions,
 )
 from .wire import (
@@ -229,6 +232,7 @@ class WorkerOperations:
 
         value = serialize_worker_register_value(
             task_types,
+            worker_type=int(opts.worker_type),
             max_concurrency=opts.concurrency,
             metadata=opts.metadata,
             machine_id=opts.machine_id,
@@ -246,24 +250,57 @@ class WorkerOperations:
         worker_id: str,
         current_load: int = 0,
         namespace: str | None = None,
-    ) -> None:
+    ) -> WorkerStatus:
         """Send a worker heartbeat to keep the registration alive.
 
         Args:
             worker_id: Worker identifier.
             current_load: Current number of active tasks.
             namespace: Optional namespace override.
+
+        Returns:
+            The worker's status on the server: DRAINING once it is drained.
+
+        Raises:
+            NotFoundError: The server doesn't know the worker (it never
+                registered, or was removed after going silent).
         """
         import struct
 
         ns = self._client.get_namespace(namespace)
         value = struct.pack("<I", current_load)
-        await self._client._send_and_check(
+        response = await self._client._send_and_check(
             OpCode.WORKER_HEARTBEAT,
             ns,
             worker_id.encode("utf-8"),
             value,
         )
+        if len(response.data) != 1:
+            raise ValueError(f"heartbeat answer is {len(response.data)} bytes, want 1")
+        return WorkerStatus(response.data[0])
+
+    async def drain(self, worker_id: str, namespace: str | None = None) -> None:
+        """Mark a worker draining: the server gives it no more work, and
+        its heartbeats answer DRAINING until it deregisters.
+
+        Raises:
+            NotFoundError: The server doesn't know the worker.
+        """
+        await self._client._send_and_check(
+            OpCode.WORKER_DRAIN,
+            self._client.get_namespace(namespace),
+            worker_id.encode("utf-8"),
+            b"",
+        )
+
+    async def deregister(self, worker_id: str, namespace: str | None = None) -> None:
+        """Remove a worker from the registry."""
+        await self._client._send_and_check(
+            OpCode.WORKER_DEREGISTER,
+            self._client.get_namespace(namespace),
+            worker_id.encode("utf-8"),
+            b"",
+        )
 
     async def await_task(
         self,
@@ -412,8 +449,23 @@ class WorkerOperations:
             OpCode.WORKER_LIST,
             namespace,
             b"",
-            encode_worker_list_body(opts.limit, opts.cursor),
+            encode_worker_list_body(opts.limit, opts.cursor, opts.action),
         )
 
         workers, cursor = decode_worker_list_answer(response.data)
         return WorkerListResult(workers=workers, cursor=cursor)
+
+    async def info(self, worker_id: str, namespace: str | None = None) -> WorkerDetails:
+        """A worker's record: its row, as :meth:`list` shows it, the metadata
+        it registered with, and its processes with their run and fail counts.
+
+        Raises:
+            NotFoundError: The server doesn't know the worker.
+        """
+        response = await self._client._send_and_check(
+            OpCode.WORKER_INFO,
+            self._client.get_namespace(namespace),
+            worker_id.encode("utf-8"),
+            b"",
+        )
+        return decode_worker_info_answer(response.data)

flo/client.py

@@ -15,6 +15,7 @@ from .exceptions import (
     NotConnectedError,
     ProtocolError,
     RequestTimeoutError,
+    ServerError,
     UnexpectedEofError,
     raise_for_status,
 )
@@ -23,7 +24,7 @@ from .types import _DEFAULT_WORKER_BLOCK_MS, HEADER_SIZE, OpCode, OptionTag, Sta
 from .wire import (
     OptionsIterator,
     RawResponse,
-    compute_crc32,
+    compute_crc32c,
     parse_response_header,
     serialize_request,
 )
@@ -387,7 +388,7 @@ class FloClient:
                 except asyncio.TimeoutError as e:
                     raise RequestTimeoutError(f"No response to {op_code.name}") from e
 
-                status, data_len, resp_request_id, crc = parse_response_header(header_data)
+                status, data_len, resp_request_id, crc, stale = parse_response_header(header_data)
 
                 if data_len > 0:
                     try:
@@ -406,10 +407,10 @@ class FloClient:
                 else:
                     response_data = b""
 
-                computed_crc = compute_crc32(header_data, response_data)
+                computed_crc = compute_crc32c(header_data, response_data)
                 if computed_crc != crc:
                     raise InvalidChecksumError(
-                        f"CRC32 mismatch: 0x{computed_crc:08X} != 0x{crc:08X}"
+                        f"CRC32C mismatch: 0x{computed_crc:08X} != 0x{crc:08X}"
                     )
                 if resp_request_id == 0:
                     # The server answers a request it could not parse with
@@ -435,7 +436,9 @@ class FloClient:
             if self._debug:
                 logger.debug(f"[flo] <- {status.name} {len(response_data)} bytes")
 
-            return RawResponse(status=status, data=response_data, request_id=resp_request_id)
+            return RawResponse(
+                status=status, data=response_data, request_id=resp_request_id, stale=stale
+            )
 
     async def _send_and_check(
         self,
@@ -471,7 +474,11 @@ class FloClient:
         if allow_not_found and response.status == StatusCode.NOT_FOUND:
             return response
 
-        raise_for_status(response.status, response.data)
+        try:
+            raise_for_status(response.status, response.data)
+        except ServerError as e:
+            e.stale = response.stale
+            raise
         return response  # Unreachable, but makes type checker happy
 
     def get_namespace(self, override: str | None) -> str:

flo/exceptions.py

@@ -101,7 +101,7 @@ class TableMismatchError(ProtocolError):
 
 
 class InvalidChecksumError(ProtocolError):
-    """CRC32 checksum validation failed."""
+    """A frame's CRC32C checksum didn't match."""
 
     pass
 
@@ -166,6 +166,9 @@ class ServerError(FloError):
     reason: Reason = Reason.UNCLASSIFIED
     # Whether the request took effect; unknown unless the server said.
     ran: Ran = Ran.UNKNOWN
+    # A halted shard answered, from its state just before the entry it
+    # halted at.
+    stale: bool = False
 
     def __init__(self, message: str, status_code: StatusCode):
         super().__init__(message)
@@ -179,6 +182,15 @@ class NotFoundError(ServerError):
         super().__init__(message, StatusCode.NOT_FOUND)
 
 
+class StaleNotFoundError(NotFoundError):
+    """A halted shard didn't have the key, as of the entry it halted at
+    (``stale`` is True). It may exist since: a write after the halt hasn't
+    applied. kv.get and kv.json_get raise it rather than report the key
+    absent."""
+
+    stale = True
+
+
 class BadRequestError(ServerError):
     """Invalid request parameters."""
 
@@ -218,13 +230,25 @@ class UnavailableError(ServerError):
     """No leader, or the shard isn't taking writes or is offline. Retryable.
 
     The message says why; an offline shard stays unavailable until an
-    operator acts, so whether and when to retry is the caller's call.
+    operator acts, so whether and when to retry is the caller's call. With
+    reason SHARD_HALTED the shard halted and takes no writes until an
+    operator repairs it; the client doesn't retry that one.
     """
 
     def __init__(self, message: str = StatusCode.UNAVAILABLE.message()):
         super().__init__(message, StatusCode.UNAVAILABLE)
 
 
+class OutcomePendingError(ServerError):
+    """The write committed but hasn't applied: its shard halted at or before
+    it (reason SHARD_HALTED, ran UNKNOWN). The message says whether it
+    applies when the shard resumes. Never retried: resending it would write
+    it twice, or halt again."""
+
+    def __init__(self, message: str = StatusCode.OUTCOME_PENDING.message()):
+        super().__init__(message, StatusCode.OUTCOME_PENDING)
+
+
 class InternalServerError(ServerError):
     """Internal server error. Not retryable: the server uses it for a write
     that committed but wasn't applied, which must not be resent."""
@@ -292,6 +316,7 @@ def _error_for(status: StatusCode, message: str) -> ServerError:
         StatusCode.UNAUTHORIZED: UnauthorizedError,
         StatusCode.OVERLOADED: OverloadedError,
         StatusCode.UNAVAILABLE: UnavailableError,
+        StatusCode.OUTCOME_PENDING: OutcomePendingError,
         StatusCode.RATE_LIMITED: RateLimitedError,
         StatusCode.INTERNAL_ERROR: InternalServerError,
         StatusCode.ERROR_GENERIC: GenericServerError,

flo/kv.py

@@ -6,7 +6,12 @@ Key-value store operations for Flo client.
 import json
 from typing import TYPE_CHECKING, Any
 
-from .exceptions import IncompleteResponseError, ServerError, raise_for_status
+from .exceptions import (
+    IncompleteResponseError,
+    ServerError,
+    StaleNotFoundError,
+    raise_for_status,
+)
 from .kv_body import (
     decode_delete_answer,
     decode_empty_answer,
@@ -36,10 +41,12 @@ from .kv_body import (
 )
 from .types import (
     DeleteOptions,
+    ExistsResult,
     GetJsonResult,
     GetOptions,
     GetResult,
     HistoryOptions,
+    HistoryResult,
     IncrResult,
     KVExistsOptions,
     KVExpireOptions,
@@ -53,7 +60,6 @@ from .types import (
     Reason,
     ScanOptions,
     ScanResult,
-    VersionEntry,
 )
 from .wire import RawResponse
 
@@ -66,17 +72,24 @@ def _key(key: str | bytes) -> bytes:
     return key.encode("utf-8") if isinstance(key, str) else key
 
 
-def _absent(response: RawResponse) -> bool:
+def _absent(response: RawResponse, *, stale_raises: bool = False) -> bool:
     """Whether a not-found answer says the thing is absent; any other not-found
-    (no such transaction) is raised."""
+    (no such transaction) is raised. With stale_raises, an absent key on a
+    stale answer raises StaleNotFoundError: absent as of the halt is not
+    absent."""
     if not response.is_not_found():
         return False
     try:
         raise_for_status(response.status, response.data)
     except ServerError as e:
-        if e.reason == Reason.NOT_FOUND:
-            return True
-        raise
+        e.stale = response.stale
+        if e.reason != Reason.NOT_FOUND:
+            raise
+        if stale_raises and response.stale:
+            stale = StaleNotFoundError(str(e))
+            stale.reason, stale.ran = e.reason, e.ran
+            raise stale from None
+        return True
     return False  # pragma: no cover - raise_for_status always raises here
 
 
@@ -87,9 +100,11 @@ async def kv_get(
     response = await client._send_and_check(
         OpCode.KV_GET, namespace, _key(key), encode_get_body(opts, txn), allow_not_found=True
     )
-    if _absent(response):
+    if _absent(response, stale_raises=True):
         return None
-    return decode_get_answer(response.data)
+    result = decode_get_answer(response.data)
+    result.stale = response.stale
+    return result
 
 
 async def kv_set(
@@ -163,12 +178,14 @@ async def kv_exists(
     key: str | bytes,
     opts: KVExistsOptions,
     txn: int | None = None,
-) -> bool:
+) -> ExistsResult:
     """kv.exists, inside transaction ``txn`` when given."""
     response = await client._send_and_check(
         OpCode.KV_EXISTS, namespace, _key(key), encode_exists_body(opts.routing_key, txn)
     )
-    return decode_exists_answer(response.data)[0]
+    result = decode_exists_answer(response.data)
+    result.stale = response.stale
+    return result
 
 
 class KVOperations:
@@ -184,8 +201,14 @@ class KVOperations:
     ) -> GetResult | None:
         """Get value and version for a key.
 
-        Returns ``GetResult(value, version)``, or ``None`` if the key is not
-        found, also when a wait (``wait_ms``) ends without it.
+        Returns ``GetResult(value, version, expires_at_ms)``, or ``None`` if
+        the key is not found, also when a wait (``wait_ms``) ends without it.
+        ``expires_at_ms`` is when the key's TTL runs out (unix ms), ``None``
+        when it has none. ``stale`` says a halted shard answered.
+
+        Raises:
+            StaleNotFoundError: A halted shard had no such key as of the entry
+                it halted at; it may exist since.
 
         Example:
             result = await client.kv.get("user:123")
@@ -281,6 +304,8 @@ class KVOperations:
                 )
                 if next_index == 0:
                     raise IncompleteResponseError("kv.mget answered none of the keys asked")
+                for e in entries:
+                    e.stale = response.stale
                 out.extend(entries)
                 done += len(entries)
         return out
@@ -293,13 +318,14 @@ class KVOperations:
         """List the names of keys with a prefix ("" for all), a page at a time.
 
         Example:
-            result = await client.kv.scan("user:")
-            for name in result.names:
-                print(name)
-
-            # Next pages
-            while result.cursor is not None:
-                result = await client.kv.scan("user:", ScanOptions(cursor=result.cursor))
+            opts = ScanOptions()
+            while True:
+                result = await client.kv.scan("user:", opts)
+                for name in result.names:
+                    print(name)
+                if result.cursor is None or not result.names:
+                    break
+                opts = ScanOptions(cursor=result.cursor)
         """
         opts = options or ScanOptions()
         response = await self._client._send_and_check(
@@ -309,31 +335,41 @@ class KVOperations:
             encode_scan_body(opts.limit, opts.cursor),
         )
         names, cursor = decode_scan_answer(response.data)
-        return ScanResult(names=names, cursor=cursor)
+        return ScanResult(names=names, cursor=cursor, stale=response.stale)
 
     async def history(
         self,
         key: str | bytes,
         options: HistoryOptions | None = None,
-    ) -> list[VersionEntry]:
-        """A key's versions, newest first, each with when it was written; empty
-        for a missing key.
+    ) -> HistoryResult:
+        """A page of a key's versions, newest first, each with when it was
+        written and whether it was a delete (``deleted``, with an empty
+        value); empty for a missing key. ``more`` says older kept versions
+        follow: ask again with ``before_version`` set to the last version of
+        this page.
 
         Example:
-            for entry in await client.kv.history("user:123", HistoryOptions(limit=10)):
-                print(f"v{entry.version} at {entry.at_ms}: {entry.value}")
+            opts = HistoryOptions(limit=10)
+            while True:
+                page = await client.kv.history("user:123", opts)
+                for entry in page.versions:
+                    print(f"v{entry.version} at {entry.at_ms}: {entry.value}")
+                if not page.more or not page.versions:
+                    break
+                opts = HistoryOptions(limit=10, before_version=page.versions[-1].version)
         """
         opts = options or HistoryOptions()
         response = await self._client._send_and_check(
             OpCode.KV_HISTORY,
             self._client.get_namespace(opts.namespace),
             _key(key),
-            encode_history_body(opts.limit, opts.routing_key),
+            encode_history_body(opts.limit, opts.before_version, opts.routing_key),
             allow_not_found=True,
         )
         if _absent(response):
-            return []
-        return decode_history_answer(response.data)
+            return HistoryResult(versions=[], more=False, stale=response.stale)
+        versions, more = decode_history_answer(response.data)
+        return HistoryResult(versions=versions, more=more, stale=response.stale)
 
     # ── Counters, TTL, exists, JSON paths ──────────────────────────────
 
@@ -373,8 +409,8 @@ class KVOperations:
         self,
         key: str | bytes,
         options: KVExistsOptions | None = None,
-    ) -> bool:
-        """Return ``True`` if ``key`` is present without transferring its value."""
+    ) -> ExistsResult:
+        """Whether ``key`` is present, and its version, without transferring its value."""
         opts = options or KVExistsOptions()
         return await kv_exists(self._client, self._client.get_namespace(opts.namespace), key, opts)
 
@@ -388,7 +424,8 @@ class KVOperations:
 
         Returns a :class:`GetJsonResult` whose ``value`` is the parsed JSON,
         with the document's current version, or ``None`` if the key or path
-        is missing.
+        is missing. A missing key on a halted shard's answer raises
+        :class:`StaleNotFoundError` instead.
         """
         opts = options or KVJsonOptions()
         response = await self._client._send_and_check(
@@ -398,10 +435,12 @@ class KVOperations:
             encode_json_get_body(path, opts.routing_key),
             allow_not_found=True,
         )
-        if _absent(response):
+        if _absent(response, stale_raises=True):
             return None
         version, document = decode_json_get_answer(response.data)
-        return GetJsonResult(value=json.loads(document.decode("utf-8")), version=version)
+        return GetJsonResult(
+            value=json.loads(document.decode("utf-8")), version=version, stale=response.stale
+        )
 
     async def json_set(
         self,