flo-python 0.1.0.dev4__py3-none-any.whl → 0.1.2__py3-none-any.whl

flo/kv_txn.py

@@ -0,0 +1,237 @@
+"""Flo KV Per-Shard Transactions
+
+Transactions are pinned to a single partition, chosen by the routing key
+passed to ``begin``; every op inside the transaction is routed by it.
+
+Caps (server-enforced):
+    * 256 ops per transaction
+    * 1 MiB total payload across buffered writes
+
+The following operations are NOT supported inside a transaction and raise
+:class:`TxnUnsupportedOpError` without a server round-trip: ``scan``,
+``mget``, ``json_get``, ``json_set``, ``json_del``, ``history``.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import dataclasses
+from typing import TYPE_CHECKING, Any
+
+from .exceptions import ValidationError
+from .kv import kv_delete, kv_exists, kv_expire, kv_get, kv_incr, kv_set
+from .kv_body import (
+    decode_empty_answer,
+    decode_txn_begin_answer,
+    decode_txn_commit_answer,
+    encode_txn_body,
+)
+from .types import (
+    DeleteOptions,
+    GetOptions,
+    GetResult,
+    KVCommitResult,
+    KVExistsOptions,
+    KVExpireOptions,
+    KVIncrOptions,
+    OpCode,
+    PutOptions,
+    PutResult,
+)
+
+if TYPE_CHECKING:
+    from .client import FloClient
+
+
+class TxnUnsupportedOpError(Exception):
+    """Raised when an operation is not allowed inside a KV transaction."""
+
+
+class TxnFinishedError(Exception):
+    """Raised when an operation is attempted on a closed transaction."""
+
+
+class Transaction:
+    """Per-shard KV transaction handle.
+
+    Operations are buffered on the server's pinned shard until :meth:`commit`
+    or :meth:`rollback` is called. Use as an async context manager for
+    automatic rollback on exception::
+
+        async with await client.kv.begin("user:123") as txn:
+            await txn.put("user:123:name", b"Jane")
+            await txn.incr("user:123:visits", 1)
+            result = await txn.commit()
+            print(result.commit_index)
+    """
+
+    def __init__(
+        self,
+        client: FloClient,
+        namespace: str,
+        routing_key: str,
+        txn_id: int,
+        pinned_hash: int,
+    ):
+        self._client = client
+        self._namespace = namespace
+        self._routing_key = routing_key
+        self._txn_id = txn_id
+        self._pinned_hash = pinned_hash
+        self._done = False
+
+    @property
+    def id(self) -> int:
+        """The server-assigned transaction id."""
+        return self._txn_id
+
+    @property
+    def pinned_hash(self) -> int:
+        """The partition hash this transaction is bound to."""
+        return self._pinned_hash
+
+    def _check_alive(self) -> None:
+        if self._done:
+            raise TxnFinishedError("transaction already committed or rolled back")
+
+    async def put(
+        self,
+        key: str | bytes,
+        value: bytes,
+        options: PutOptions | None = None,
+    ) -> PutResult:
+        """Buffer a put inside the transaction; it has no version until commit (0)."""
+        self._check_alive()
+        opts = dataclasses.replace(options or PutOptions(), routing_key=self._routing_key)
+        return await kv_set(self._client, self._namespace, key, value, opts, self._txn_id)
+
+    async def get(self, key: str | bytes) -> GetResult | None:
+        """Read a key inside the transaction (sees buffered writes)."""
+        self._check_alive()
+        return await kv_get(
+            self._client,
+            self._namespace,
+            key,
+            GetOptions(routing_key=self._routing_key),
+            self._txn_id,
+        )
+
+    async def delete(
+        self,
+        key: str | bytes,
+        options: DeleteOptions | None = None,
+    ) -> None:
+        """Buffer a delete inside the transaction; whether the key existed is unknown."""
+        self._check_alive()
+        opts = dataclasses.replace(options or DeleteOptions(), routing_key=self._routing_key)
+        await kv_delete(self._client, self._namespace, key, opts, self._txn_id)
+
+    async def incr(self, key: str | bytes, delta: int = 1) -> None:
+        """Buffer an atomic counter increment; its value is unknown until commit."""
+        self._check_alive()
+        opts = KVIncrOptions(delta=delta, routing_key=self._routing_key)
+        await kv_incr(self._client, self._namespace, key, opts, self._txn_id)
+
+    async def expire(
+        self,
+        key: str | bytes,
+        ttl_ms: int,
+        options: KVExpireOptions | None = None,
+    ) -> None:
+        """Buffer setting the TTL on a key, in milliseconds; ``ttl_ms=0`` clears it."""
+        self._check_alive()
+        opts = dataclasses.replace(options or KVExpireOptions(), routing_key=self._routing_key)
+        await kv_expire(self._client, self._namespace, key, ttl_ms, opts, self._txn_id)
+
+    async def exists(self, key: str | bytes) -> bool:
+        """Check key existence inside the transaction (sees buffered writes)."""
+        self._check_alive()
+        opts = KVExistsOptions(routing_key=self._routing_key)
+        return await kv_exists(self._client, self._namespace, key, opts, self._txn_id)
+
+    # ── Disallowed inside a transaction ───────────────────────────────
+
+    async def scan(self, *args: Any, **kwargs: Any) -> None:  # pragma: no cover
+        raise TxnUnsupportedOpError("scan is not supported inside a KV transaction")
+
+    async def mget(self, *args: Any, **kwargs: Any) -> None:  # pragma: no cover
+        raise TxnUnsupportedOpError("mget is not supported inside a KV transaction")
+
+    async def json_get(self, *args: Any, **kwargs: Any) -> None:  # pragma: no cover
+        raise TxnUnsupportedOpError("json_get is not supported inside a KV transaction")
+
+    async def json_set(self, *args: Any, **kwargs: Any) -> None:  # pragma: no cover
+        raise TxnUnsupportedOpError("json_set is not supported inside a KV transaction")
+
+    async def json_del(self, *args: Any, **kwargs: Any) -> None:  # pragma: no cover
+        raise TxnUnsupportedOpError("json_del is not supported inside a KV transaction")
+
+    async def history(self, *args: Any, **kwargs: Any) -> None:  # pragma: no cover
+        raise TxnUnsupportedOpError("history is not supported inside a KV transaction")
+
+    # ── Lifecycle ──────────────────────────────────────────────────────
+
+    async def commit(self) -> KVCommitResult:
+        """Atomically apply all buffered operations.
+
+        After commit returns the transaction is closed and further operations
+        raise :class:`TxnFinishedError`.
+        """
+        self._check_alive()
+        self._done = True
+        response = await self._client._send_and_check(
+            OpCode.KV_COMMIT_TXN, self._namespace, b"", encode_txn_body(self._txn_id)
+        )
+        commit_index, op_count = decode_txn_commit_answer(response.data)
+        return KVCommitResult(commit_index=commit_index, op_count=op_count)
+
+    async def rollback(self) -> None:
+        """Discard the buffered operations without committing.
+
+        Idempotent: calling rollback after commit (or vice versa) is a no-op.
+        """
+        if self._done:
+            return
+        self._done = True
+        response = await self._client._send_and_check(
+            OpCode.KV_ROLLBACK_TXN, self._namespace, b"", encode_txn_body(self._txn_id)
+        )
+        decode_empty_answer("kv.txn_rollback", response.data)
+
+    async def __aenter__(self) -> Transaction:
+        return self
+
+    async def __aexit__(self, exc_type: object, exc: object, tb: object) -> None:
+        if not self._done:
+            # Best-effort cleanup; surface the original exception.
+            with contextlib.suppress(Exception):
+                await self.rollback()
+
+
+async def begin(
+    client: FloClient,
+    namespace: str,
+    routing_key: str,
+) -> Transaction:
+    """Open a new per-shard KV transaction: its routing key is the request's key."""
+    if not routing_key:
+        raise ValidationError("a transaction needs a routing key")
+    response = await client._send_and_check(
+        OpCode.KV_BEGIN_TXN, namespace, routing_key.encode("utf-8"), b""
+    )
+    txn_id, pinned_hash = decode_txn_begin_answer(response.data)
+    return Transaction(
+        client=client,
+        namespace=namespace,
+        routing_key=routing_key,
+        txn_id=txn_id,
+        pinned_hash=pinned_hash,
+    )
+
+
+__all__ = [
+    "Transaction",
+    "TxnUnsupportedOpError",
+    "TxnFinishedError",
+    "begin",
+]

flo/processing.py

@@ -1,7 +1,6 @@
 """Flo Processing Operations
 
-Stream processing operations: submit, status, list, stop, cancel,
-savepoint, restore, rescale, sync.
+Stream processing operations: submit, status, list, stop, cancel, sync.
 """
 
 from __future__ import annotations
@@ -17,9 +16,6 @@ from .types import (
     ProcessingCancelOptions,
     ProcessingListEntry,
     ProcessingListOptions,
-    ProcessingRescaleOptions,
-    ProcessingRestoreOptions,
-    ProcessingSavepointOptions,
     ProcessingStatusOptions,
     ProcessingStatusResult,
     ProcessingStopOptions,
@@ -126,58 +122,6 @@ class ProcessingOperations:
             b"",
         )
 
-    async def savepoint(
-        self, job_id: str, options: ProcessingSavepointOptions | None = None
-    ) -> str:
-        """Trigger a savepoint for a processing job. Returns the savepoint ID."""
-        opts = options or ProcessingSavepointOptions()
-        namespace = self._client.get_namespace(opts.namespace)
-
-        resp = await self._client._send_and_check(
-            OpCode.PROCESSING_SAVEPOINT,
-            namespace,
-            job_id.encode("utf-8"),
-            b"",
-        )
-
-        return resp.data.decode("utf-8")
-
-    async def restore(
-        self,
-        job_id: str,
-        savepoint_id: str,
-        options: ProcessingRestoreOptions | None = None,
-    ) -> None:
-        """Restore a processing job from a savepoint."""
-        opts = options or ProcessingRestoreOptions()
-        namespace = self._client.get_namespace(opts.namespace)
-
-        await self._client._send_and_check(
-            OpCode.PROCESSING_RESTORE,
-            namespace,
-            job_id.encode("utf-8"),
-            savepoint_id.encode("utf-8"),
-        )
-
-    async def rescale(
-        self,
-        job_id: str,
-        parallelism: int,
-        options: ProcessingRescaleOptions | None = None,
-    ) -> None:
-        """Change the parallelism of a processing job."""
-        opts = options or ProcessingRescaleOptions()
-        namespace = self._client.get_namespace(opts.namespace)
-
-        value = struct.pack("<I", parallelism)
-
-        await self._client._send_and_check(
-            OpCode.PROCESSING_RESCALE,
-            namespace,
-            job_id.encode("utf-8"),
-            value,
-        )
-
     # =========================================================================
     # Declarative Sync
     # =========================================================================
@@ -224,7 +168,7 @@ def _parse_processing_status(data: bytes) -> ProcessingStatusResult:
     """Parse binary wire format for processing job status.
 
     Wire format: [job_id_len:u16][job_id][name_len:u16][name][status:u8]
-                 [parallelism:u32][batch_size:u32][records_processed:u64][created_at:i64]
+                 [batch_size:u32][records_processed:u64][created_at:i64]
     """
     pos = 0
 
@@ -252,9 +196,6 @@ def _parse_processing_status(data: bytes) -> ProcessingStatusResult:
         else f"unknown({status_byte})"
     )
 
-    (parallelism,) = struct.unpack_from("<I", data, pos)
-    pos += 4
-
     (batch_size,) = struct.unpack_from("<I", data, pos)
     pos += 4
 
@@ -267,7 +208,6 @@ def _parse_processing_status(data: bytes) -> ProcessingStatusResult:
         job_id=job_id,
         name=name,
         status=status_str,
-        parallelism=parallelism,
         batch_size=batch_size,
         records_processed=records_processed,
         created_at=created_at,
@@ -278,7 +218,7 @@ def _parse_processing_list(data: bytes) -> list[ProcessingListEntry]:
     """Parse binary wire format for processing job list.
 
     Wire format: [count:u32]([name_len:u16][name][job_id_len:u16][job_id]
-                 [status_len:u16][status][parallelism:u32][created_at:i64])*
+                 [status_len:u16][status][created_at:i64])*
     """
     if len(data) < 4:
         return []
@@ -305,9 +245,6 @@ def _parse_processing_list(data: bytes) -> list[ProcessingListEntry]:
         job_id = read_str()
         status = read_str()
 
-        (parallelism,) = struct.unpack_from("<I", data, pos)
-        pos += 4
-
         (created_at,) = struct.unpack_from("<q", data, pos)
         pos += 8
 
@@ -316,7 +253,6 @@ def _parse_processing_list(data: bytes) -> list[ProcessingListEntry]:
                 name=name,
                 job_id=job_id,
                 status=status,
-                parallelism=parallelism,
                 created_at=created_at,
             )
         )

flo/queue.py

@@ -16,7 +16,6 @@ from .types import (
     OpCode,
     OptionTag,
     PeekOptions,
-    TouchOptions,
 )
 from .wire import OptionsBuilder, parse_dequeue_response, parse_enqueue_response, serialize_seqs
 
@@ -41,7 +40,7 @@ class QueueOperations:
         Args:
             queue: Queue name.
             payload: Message payload.
-            options: Optional enqueue options (priority, delay, dedup_key).
+            options: Optional enqueue options (priority).
 
         Returns:
             Sequence number of the enqueued message.
@@ -50,14 +49,8 @@ class QueueOperations:
             # Simple enqueue
             seq = await client.queue.enqueue("tasks", b'{"task": "process"}')
 
-            # With priority (higher = more urgent)
+            # With priority (0-255, lower is taken first; default 0)
             seq = await client.queue.enqueue("tasks", payload, EnqueueOptions(priority=10))
-
-            # With delay
-            seq = await client.queue.enqueue("tasks", payload, EnqueueOptions(delay_ms=60000))
-
-            # With deduplication
-            seq = await client.queue.enqueue("tasks", payload, EnqueueOptions(dedup_key="task-123"))
         """
         opts = options or EnqueueOptions()
         namespace = self._client.get_namespace(opts.namespace)
@@ -68,12 +61,6 @@ class QueueOperations:
         if opts.priority != 0:
             builder.add_u8(OptionTag.PRIORITY, opts.priority)
 
-        if opts.delay_ms is not None:
-            builder.add_u64(OptionTag.DELAY_MS, opts.delay_ms)
-
-        if opts.dedup_key is not None:
-            builder.add_bytes(OptionTag.DEDUP_KEY, opts.dedup_key.encode("utf-8"))
-
         response = await self._client._send_and_check(
             OpCode.QUEUE_ENQUEUE,
             namespace,
@@ -92,10 +79,14 @@ class QueueOperations:
     ) -> DequeueResult:
         """Dequeue messages from a queue.
 
+        Queues are currently at-most-once: the server acknowledges each
+        message as it hands it out, so it is not redelivered in normal
+        operation, even if the consumer crashes.
+
         Args:
             queue: Queue name.
             count: Maximum number of messages to dequeue.
-            options: Optional dequeue options (visibility_timeout, block_ms).
+            options: Optional dequeue options (block_ms).
 
         Returns:
             DequeueResult containing list of messages.
@@ -105,19 +96,12 @@ class QueueOperations:
             result = await client.queue.dequeue("tasks", 10)
             for msg in result.messages:
                 process(msg.payload)
-                await client.queue.ack("tasks", [msg.seq])
 
             # With long polling (wait up to 30s for messages)
             result = await client.queue.dequeue(
                 "tasks", 10,
                 DequeueOptions(block_ms=30000)
             )
-
-            # With custom visibility timeout
-            result = await client.queue.dequeue(
-                "tasks", 10,
-                DequeueOptions(visibility_timeout_ms=60000)
-            )
         """
         opts = options or DequeueOptions()
         namespace = self._client.get_namespace(opts.namespace)
@@ -126,9 +110,6 @@ class QueueOperations:
         builder = OptionsBuilder()
         builder.add_u32(OptionTag.COUNT, count)
 
-        if opts.visibility_timeout_ms is not None:
-            builder.add_u32(OptionTag.VISIBILITY_TIMEOUT_MS, opts.visibility_timeout_ms)
-
         if opts.block_ms is not None:
             builder.add_u32(OptionTag.BLOCK_MS, opts.block_ms)
 
@@ -148,21 +129,15 @@ class QueueOperations:
         seqs: list[int],
         options: AckOptions | None = None,
     ) -> None:
-        """Acknowledge messages as successfully processed.
+        """Acknowledge messages.
+
+        Queues are currently at-most-once: dequeue already acknowledges each
+        message it hands out, so this has no effect on a dequeued message.
 
         Args:
             queue: Queue name.
             seqs: Sequence numbers of messages to acknowledge.
             options: Optional ack options.
-
-        Example:
-            result = await client.queue.dequeue("tasks", 10)
-            for msg in result.messages:
-                try:
-                    process(msg.payload)
-                    await client.queue.ack("tasks", [msg.seq])
-                except Exception:
-                    await client.queue.nack("tasks", [msg.seq])
         """
         if not seqs:
             return
@@ -185,19 +160,16 @@ class QueueOperations:
         seqs: list[int],
         options: NackOptions | None = None,
     ) -> None:
-        """Negative acknowledge messages (retry or send to DLQ).
+        """Negatively acknowledge messages.
+
+        Queues are currently at-most-once: dequeue already acknowledges each
+        message it hands out, so this has no effect on a dequeued message and
+        does not retry it.
 
         Args:
             queue: Queue name.
             seqs: Sequence numbers of messages to nack.
-            options: Optional nack options (to_dlq).
-
-        Example:
-            # Retry the message
-            await client.queue.nack("tasks", [msg.seq])
-
-            # Send to DLQ (don't retry)
-            await client.queue.nack("tasks", [msg.seq], NackOptions(to_dlq=True))
+            options: Optional nack options.
         """
         if not seqs:
             return
@@ -205,12 +177,6 @@ class QueueOperations:
         opts = options or NackOptions()
         namespace = self._client.get_namespace(opts.namespace)
 
-        # Build TLV options
-        builder = OptionsBuilder()
-
-        if opts.to_dlq:
-            builder.add_u8(OptionTag.SEND_TO_DLQ, 1)
-
         value = serialize_seqs(seqs)
 
         await self._client._send_and_check(
@@ -218,7 +184,6 @@ class QueueOperations:
             namespace,
             queue.encode("utf-8"),
             value,
-            builder.build(),
         )
 
     async def dlq_list(
@@ -228,31 +193,29 @@ class QueueOperations:
     ) -> DequeueResult:
         """List messages in the Dead Letter Queue.
 
+        Queues are currently at-most-once, so messages don't reach the DLQ in
+        normal use.
+
         Args:
             queue: Queue name.
-            options: Optional DLQ list options (limit).
+            options: Optional DLQ list options (namespace).
 
         Returns:
             DequeueResult containing list of DLQ messages.
 
         Example:
-            result = await client.queue.dlq_list("tasks", DlqListOptions(limit=100))
+            result = await client.queue.dlq_list("tasks")
             for msg in result.messages:
                 print(f"Failed message {msg.seq}: {msg.payload}")
         """
         opts = options or DlqListOptions()
         namespace = self._client.get_namespace(opts.namespace)
 
-        # Build TLV options
-        builder = OptionsBuilder()
-        builder.add_u32(OptionTag.LIMIT, opts.limit)
-
         response = await self._client._send_and_check(
             OpCode.QUEUE_DLQ_LIST,
             namespace,
             queue.encode("utf-8"),
             b"",
-            builder.build(),
         )
 
         return parse_dequeue_response(response.data)
@@ -265,6 +228,9 @@ class QueueOperations:
     ) -> None:
         """Move messages from DLQ back to the main queue.
 
+        Queues are currently at-most-once, so messages don't reach the DLQ in
+        normal use.
+
         Args:
             queue: Queue name.
             seqs: Sequence numbers of DLQ messages to requeue.
@@ -332,45 +298,3 @@ class QueueOperations:
         )
 
         return parse_dequeue_response(response.data)
-
-    async def touch(
-        self,
-        queue: str,
-        seqs: list[int],
-        options: TouchOptions | None = None,
-    ) -> None:
-        """Renew lease on messages to prevent visibility timeout.
-
-        Call this periodically during long-running processing to prevent
-        the message from becoming visible again before you're done.
-
-        Args:
-            queue: Queue name.
-            seqs: Sequence numbers of messages to touch.
-            options: Optional touch options.
-
-        Example:
-            result = await client.queue.dequeue("tasks", 1)
-            msg = result.messages[0]
-
-            # Long running task - renew lease periodically
-            for chunk in process_in_chunks(msg.payload):
-                await process_chunk(chunk)
-                await client.queue.touch("tasks", [msg.seq])
-
-            await client.queue.ack("tasks", [msg.seq])
-        """
-        if not seqs:
-            return
-
-        opts = options or TouchOptions()
-        namespace = self._client.get_namespace(opts.namespace)
-
-        value = serialize_seqs(seqs)
-
-        await self._client._send_and_check(
-            OpCode.QUEUE_TOUCH,
-            namespace,
-            queue.encode("utf-8"),
-            value,
-        )