dex-python-sdk 0.2.1__tar.gz → 0.2.2__tar.gz

{dex_python_sdk-0.2.1 → dex_python_sdk-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dex-python-sdk
-Version: 0.2.1
+Version: 0.2.2
 Requires-Dist: grpcio>=1.83.0
 Requires-Dist: grpcio-status>=1.83.0
 Requires-Dist: protobuf>=7.35.1
@@ -22,7 +22,7 @@ Python SDK for [Dex workflow engine](https://github.com/superdurable/dex)
 ## New user contracts
 
 The rewrite targets Python 3.11+ and exposes strongly typed workflow contracts
-from `dex`. This phase includes definitions, attributes, channels, waits,
+from `dex`. This phase includes definitions, attributes, channels, streams, waits,
 decisions, codecs, registry validation, synchronous client calls, and synchronous
 worker handlers. Python owns its gRPC Client and Worker transport;
 the shared Rust Core is used only for BlobCache.
@@ -34,6 +34,7 @@ import dex
 
 counter = dex.Attribute("counter", int)
 counters_by_region = dex.AttributeMap("counters-by-region", int)
+progress = dex.Stream("progress", str, 10 * 1024 * 1024)
 
 class Run(dex.Step[str]):
     def wait_for(
@@ -46,6 +47,7 @@ class Run(dex.Step[str]):
     def execute(
         self, context: dex.Context, input: str
     ) -> dex.StepDecision:
+        progress.write(context, "running")
         return dex.graceful_complete(input)
 
 class CounterFlow(dex.Flow[str]):
@@ -58,7 +60,7 @@ class CounterFlow(dex.Flow[str]):
         return dex.StepList.start_step(self.run)
 
     def get_persistence_schema(self) -> dex.PersistenceSchema:
-        return dex.PersistenceSchema.of(counter, counters_by_region)
+        return dex.PersistenceSchema.of(counter, counters_by_region, progress)
 
     @dex.rpc(name="Increment")
     def increment(
@@ -73,9 +75,26 @@ registry = dex.Registry((flow,))
 Registry derives codecs from declared Python types and handler annotations.
 Built-in primitive types and dataclasses need no codec arguments. Register an
 explicit codec only for a custom encoding or a type Registry cannot derive.
-`PersistenceSchema.of(...)` accepts attributes and channels together and
+`PersistenceSchema.of(...)` accepts attributes, channels, and streams together and
 partitions them by definition type.
 
+Streams provide best-effort resumable progress messages. Their approximate byte
+budget is shared by all instances of the owning Flow type. Client keys cannot
+contain `#`; Step writes generate `runID#stepExecutionID` and allow one write per
+Stream per invocation.
+
+```python
+client.write_stream(flow_id, progress, "frontend/1", "starting")
+message = client.read_stream(
+    flow_id, progress, resume_token, timeout=timedelta(seconds=30)
+)
+resume_token = message.resume_token
+```
+
+Async Step handlers await **Stream.write**, and **AsyncClient** exposes matching
+async methods. Reads return the decoded value, resume token, creation time, and
+idempotency key.
+
 `Worker` and `AsyncWorker` synchronize all registered Indexed Attributes with
 Dex Server before opening their listener. Existing indexes return immediately;
 failure or the default two-minute deadline aborts startup. An indexed

{dex_python_sdk-0.2.1 → dex_python_sdk-0.2.2}/README.md

@@ -6,7 +6,7 @@ Python SDK for [Dex workflow engine](https://github.com/superdurable/dex)
 ## New user contracts
 
 The rewrite targets Python 3.11+ and exposes strongly typed workflow contracts
-from `dex`. This phase includes definitions, attributes, channels, waits,
+from `dex`. This phase includes definitions, attributes, channels, streams, waits,
 decisions, codecs, registry validation, synchronous client calls, and synchronous
 worker handlers. Python owns its gRPC Client and Worker transport;
 the shared Rust Core is used only for BlobCache.
@@ -18,6 +18,7 @@ import dex
 
 counter = dex.Attribute("counter", int)
 counters_by_region = dex.AttributeMap("counters-by-region", int)
+progress = dex.Stream("progress", str, 10 * 1024 * 1024)
 
 class Run(dex.Step[str]):
     def wait_for(
@@ -30,6 +31,7 @@ class Run(dex.Step[str]):
     def execute(
         self, context: dex.Context, input: str
     ) -> dex.StepDecision:
+        progress.write(context, "running")
         return dex.graceful_complete(input)
 
 class CounterFlow(dex.Flow[str]):
@@ -42,7 +44,7 @@ class CounterFlow(dex.Flow[str]):
         return dex.StepList.start_step(self.run)
 
     def get_persistence_schema(self) -> dex.PersistenceSchema:
-        return dex.PersistenceSchema.of(counter, counters_by_region)
+        return dex.PersistenceSchema.of(counter, counters_by_region, progress)
 
     @dex.rpc(name="Increment")
     def increment(
@@ -57,9 +59,26 @@ registry = dex.Registry((flow,))
 Registry derives codecs from declared Python types and handler annotations.
 Built-in primitive types and dataclasses need no codec arguments. Register an
 explicit codec only for a custom encoding or a type Registry cannot derive.
-`PersistenceSchema.of(...)` accepts attributes and channels together and
+`PersistenceSchema.of(...)` accepts attributes, channels, and streams together and
 partitions them by definition type.
 
+Streams provide best-effort resumable progress messages. Their approximate byte
+budget is shared by all instances of the owning Flow type. Client keys cannot
+contain `#`; Step writes generate `runID#stepExecutionID` and allow one write per
+Stream per invocation.
+
+```python
+client.write_stream(flow_id, progress, "frontend/1", "starting")
+message = client.read_stream(
+    flow_id, progress, resume_token, timeout=timedelta(seconds=30)
+)
+resume_token = message.resume_token
+```
+
+Async Step handlers await **Stream.write**, and **AsyncClient** exposes matching
+async methods. Reads return the decoded value, resume token, creation time, and
+idempotency key.
+
 `Worker` and `AsyncWorker` synchronize all registered Indexed Attributes with
 Dex Server before opening their listener. Existing indexes return immediately;
 failure or the default two-minute deadline aborts startup. An indexed

{dex_python_sdk-0.2.1 → dex_python_sdk-0.2.2}/dex/__init__.py

@@ -47,14 +47,14 @@ from dex.flow_info import (
 from dex.flow_options import (
     FlowTimeoutPolicy,
     IdReusePolicy,
-    TimeTravelOptions,
-    TimeTravelStepMethod,
-    TimeTravelType,
     StartFlowOptions,
-    SubFlowOptions,
-    SubFlowReusePolicy,
     StopFlowOptions,
     StopType,
+    SubFlowOptions,
+    SubFlowReusePolicy,
+    TimeTravelOptions,
+    TimeTravelStepMethod,
+    TimeTravelType,
 )
 from dex.flow_result import FlowResult, StepCompletion
 from dex.runtime_errors import (
@@ -89,8 +89,9 @@ from dex.step import (
     graceful_complete,
 )
 from dex.step_execution import StepExecutionId, TimerId
-from dex.timer import Timer
+from dex.stream import Stream, StreamMessage
 from dex.subflow import SubFlow
+from dex.timer import Timer
 from dex.wait import Wait
 from dex.worker import Worker
 from dex.worker_options import WorkerOptions, WorkerTarget
@@ -159,6 +160,8 @@ __all__ = [
     "StepDurability",
     "StepMovement",
     "StepOptions",
+    "Stream",
+    "StreamMessage",
     "StopFlowOptions",
     "StopType",
     "Timer",

{dex_python_sdk-0.2.1 → dex_python_sdk-0.2.2}/dex/_async_worker_dispatcher.py

@@ -28,10 +28,12 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
         registry: Registry,
         values: ValueMapper,
         hydrator: AsyncValueHydrator,
+        stream_writer: Callable[[pb.WriteStreamRequest], Any],
     ) -> None:
         self._registry = registry
         self._values = values
         self._async_hydrator = hydrator
+        self._stream_writer = stream_writer
 
     async def invoke_wait_for(  # type: ignore[override]
         self,
@@ -46,6 +48,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
             flow,
             request.context,
             self._values,
+            self._stream_writer,
             request.attributes,
             is_active=is_active,
         )
@@ -89,6 +92,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
             flow,
             request.context,
             self._values,
+            self._stream_writer,
             request.attributes,
             request.step_exe_locals,
             condition_results,
@@ -137,6 +141,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
             flow,
             request.context,
             self._values,
+            self._stream_writer,
             request.attributes,
             request.step_exe_locals,
             condition_results,
@@ -172,6 +177,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
             flow,
             request.context,
             self._values,
+            self._stream_writer,
             request.attributes,
             channel_infos=dict(request.channel_infos),
             is_active=is_active,

{dex_python_sdk-0.2.1 → dex_python_sdk-0.2.2}/dex/_invocation_context.py

@@ -9,6 +9,7 @@
 from __future__ import annotations
 
 from enum import Enum
+from inspect import isawaitable
 from typing import Any, Callable, Sequence, TypeVar, cast
 from urllib.parse import unquote
 
@@ -20,9 +21,12 @@ from dex.codec import Codec
 from dex.dexpb import dex_pb2 as pb
 from dex.flow import Registry, _RegisteredFlow
 from dex.flow_result import FlowResult, flow_result_from_proto
+from dex.stream import Stream
 
 ValueT = TypeVar("ValueT")
-_Definition = Attribute[Any] | AttributeMap[Any] | Channel[Any] | ChannelMap[Any]
+_Definition = (
+    Attribute[Any] | AttributeMap[Any] | Channel[Any] | ChannelMap[Any] | Stream[Any]
+)
 
 
 class InvocationMethod(Enum):
@@ -38,6 +42,7 @@ class InvocationContext:
         flow: _RegisteredFlow,
         metadata: pb.Context,
         values: ValueMapper,
+        stream_writer: Callable[[pb.WriteStreamRequest], Any],
         attributes: Sequence[pb.KV],
         locals: Sequence[pb.KV] = (),
         condition_results: pb.ConditionResults | None = None,
@@ -48,6 +53,7 @@ class InvocationContext:
         self._flow = flow
         self._metadata = metadata
         self._values = values
+        self._stream_writer = stream_writer
         self._attributes = self._map_values("Attribute", attributes)
         self._locals = self._map_values("step-execution local", locals)
         self._condition_results = condition_results
@@ -58,6 +64,7 @@ class InvocationContext:
         self.events: list[pb.KV] = []
         self.publications: list[pb.ChannelMessage] = []
         self._event_names: set[str] = set()
+        self._stream_writes: set[int] = set()
 
     @property
     def flow_id(self) -> str:
@@ -151,6 +158,48 @@ class InvocationContext:
         self._event_names.add(name)
         self.events.append(pb.KV(key=name, value=self._values.encode_dynamic(value)))
 
+    def _write_stream(
+        self,
+        definition: Stream[ValueT],
+        value: ValueT,
+    ) -> Any:
+        if self._method is InvocationMethod.RPC:
+            raise ValueError("Stream writes require a Step Context")
+        self._require_registered(definition)
+        identity = id(definition)
+        if identity in self._stream_writes:
+            raise ValueError(
+                f"Stream {definition.name} was already written by this Step execution"
+            )
+        request = pb.WriteStreamRequest(
+            flow_id=self.flow_id,
+            flow_type=self._flow.name,
+            stream_name=definition.name,
+            max_estimated_bytes=definition.max_estimated_bytes,
+            value=self._values.encode(
+                value,
+                self._values.codec(definition.value_type),
+            ),
+            idempotency_key=f"{self.run_id}#{self.step_execution_id}",
+        )
+        result = self._stream_writer(request)
+        if isawaitable(result):
+            self._stream_writes.add(identity)
+            return self._finish_async_stream_write(result, identity)
+        self._stream_writes.add(identity)
+        return None
+
+    async def _finish_async_stream_write(
+        self,
+        result: Any,
+        identity: int,
+    ) -> None:
+        try:
+            await result
+        except BaseException:
+            self._stream_writes.remove(identity)
+            raise
+
     def _get_attribute(
         self,
         definition: Attribute[ValueT] | AttributeMap[ValueT],

{dex_python_sdk-0.2.1 → dex_python_sdk-0.2.2}/dex/_worker_dispatcher.py

@@ -49,10 +49,12 @@ class WorkerDispatcher:
         registry: Registry,
         values: ValueMapper,
         hydrator: ValueHydrator,
+        stream_writer: Callable[[pb.WriteStreamRequest], Any],
     ) -> None:
         self._registry = registry
         self._values = values
         self._hydrator = hydrator
+        self._stream_writer = stream_writer
 
     def invoke_wait_for(
         self,
@@ -67,6 +69,7 @@ class WorkerDispatcher:
             flow,
             request.context,
             self._values,
+            self._stream_writer,
             request.attributes,
             is_active=is_active,
         )
@@ -112,6 +115,7 @@ class WorkerDispatcher:
             flow,
             request.context,
             self._values,
+            self._stream_writer,
             request.attributes,
             request.step_exe_locals,
             condition_results,
@@ -162,6 +166,7 @@ class WorkerDispatcher:
             flow,
             request.context,
             self._values,
+            self._stream_writer,
             request.attributes,
             request.step_exe_locals,
             condition_results,
@@ -199,6 +204,7 @@ class WorkerDispatcher:
             flow,
             request.context,
             self._values,
+            self._stream_writer,
             request.attributes,
             channel_infos=dict(request.channel_infos),
             is_active=is_active,

{dex_python_sdk-0.2.1 → dex_python_sdk-0.2.2}/dex/async_client.py

@@ -35,18 +35,19 @@ from dex.flow_info import FlowInfo, FlowStatus, SearchFlowEntry, SearchFlowsPage
 from dex.flow_options import (
     FlowTimeoutPolicy,
     IdReusePolicy,
-    TimeTravelOptions,
-    TimeTravelStepMethod,
-    TimeTravelType,
     StartFlowOptions,
     StopFlowOptions,
     StopType,
+    TimeTravelOptions,
+    TimeTravelStepMethod,
+    TimeTravelType,
     _resolve_flow_timeout_policy,
 )
 from dex.flow_result import FlowResult, flow_result_from_proto
 from dex.runtime_errors import FlowErrorType
 from dex.step import RetryPolicy, StepDurability
 from dex.step_execution import StepExecutionId, TimerId
+from dex.stream import Stream, StreamMessage
 
 InputT = TypeVar("InputT")
 OutputT = TypeVar("OutputT")
@@ -97,6 +98,7 @@ class AsyncClient:
             registry,
             self._values,
             cast(Any, self._hydrator),
+            self._service.WriteStream,
         )
         self._closed = False
 
@@ -502,6 +504,109 @@ class AsyncClient:
             "active",
         )
 
+    async def write_stream(
+        self,
+        flow_id: str,
+        stream: Stream[ValueT],
+        idempotency_key: str,
+        value: ValueT,
+    ) -> None:
+        """Await one typed best-effort Stream append with client idempotency.
+
+        Args:
+            flow_id: Logical Flow instance ID; the Flow need not exist or be active.
+            stream: Exact Stream object registered in one Flow schema.
+            idempotency_key: Non-empty key without ``#``; retained duplicates are no-ops.
+            value: Typed message to append.
+
+        Raises:
+            ValueError: If an ID is empty or the key contains the reserved separator.
+            FlowDefinitionError: If the Stream is not registered.
+            ValueMappingError: If the message cannot be encoded.
+            DexServiceError: If FlowService cannot append the message.
+        """
+        require_name(idempotency_key)
+        if "#" in idempotency_key:
+            raise ValueError("Stream client idempotency key must not contain #")
+        flow = self.registry._flow_for_stream(stream)
+        await self._call(
+            self._service.WriteStream,
+            pb.WriteStreamRequest(
+                flow_id=require_name(flow_id),
+                flow_type=flow.name,
+                stream_name=stream.name,
+                max_estimated_bytes=stream.max_estimated_bytes,
+                value=self._values.encode(
+                    value,
+                    self._values.codec(stream.value_type),
+                ),
+                idempotency_key=idempotency_key,
+            ),
+            "write_stream",
+            flow_id,
+            "none",
+        )
+
+    async def read_stream(
+        self,
+        flow_id: str,
+        stream: Stream[ValueT],
+        resume_token: str = "",
+        timeout: timedelta | None = None,
+    ) -> StreamMessage[ValueT]:
+        """Await the next retained Stream message after a resume token.
+
+        Args:
+            flow_id: Logical Flow instance ID used as the Stream instance key.
+            stream: Exact Stream object registered in one Flow schema.
+            resume_token: Previous message token, or empty for the retained head.
+            timeout: Optional non-negative server-side long-poll duration.
+
+        Returns:
+            The decoded message, next resume token, creation time, and idempotency key.
+
+        Raises:
+            LongPollTimeoutError: If no message arrives before the server wait expires.
+            FlowDefinitionError: If the Stream is not registered.
+            ValueMappingError: If the retained message cannot be decoded.
+            DexServiceError: If FlowService cannot perform the read.
+        """
+        flow = self.registry._flow_for_stream(stream)
+        response = cast(
+            pb.ReadStreamResponse,
+            await self._call(
+                self._service.ReadStream,
+                pb.ReadStreamRequest(
+                    flow_id=require_name(flow_id),
+                    flow_type=flow.name,
+                    stream_name=stream.name,
+                    resume_token=resume_token,
+                    wait_time_seconds=(
+                        0 if timeout is None else self._seconds32(timeout)
+                    ),
+                ),
+                "read_stream",
+                flow_id,
+                "none",
+            ),
+        )
+        if (
+            not response.HasField("message")
+            or not response.message.HasField("value")
+            or not response.message.HasField("created_time")
+            or not response.message.resume_token
+        ):
+            raise ValueError("Dex returned an incomplete Stream message")
+        return StreamMessage(
+            self._values.decode(
+                response.message.value,
+                self._values.codec(stream.value_type),
+            ),
+            response.message.resume_token,
+            response.message.created_time.ToDatetime(tzinfo=timezone.utc),
+            response.message.idempotency_key,
+        )
+
     async def wait_for_flow(
         self,
         flow_id: str,

{dex_python_sdk-0.2.1 → dex_python_sdk-0.2.2}/dex/async_worker.py

@@ -69,6 +69,7 @@ class AsyncWorker:
             registry,
             values,
             AsyncValueHydrator(self._flow_service, blob_cache),
+            self._flow_service.WriteStream,
         )
         self._server = grpc.aio.server()
         dex_pb2_grpc.add_WorkerServiceServicer_to_server(  # type: ignore[no-untyped-call]

{dex_python_sdk-0.2.1 → dex_python_sdk-0.2.2}/dex/client.py

@@ -35,18 +35,19 @@ from dex.flow_info import FlowInfo, FlowStatus, SearchFlowEntry, SearchFlowsPage
 from dex.flow_options import (
     FlowTimeoutPolicy,
     IdReusePolicy,
-    TimeTravelOptions,
-    TimeTravelStepMethod,
-    TimeTravelType,
     StartFlowOptions,
     StopFlowOptions,
     StopType,
+    TimeTravelOptions,
+    TimeTravelStepMethod,
+    TimeTravelType,
     _resolve_flow_timeout_policy,
 )
 from dex.flow_result import FlowResult, flow_result_from_proto
 from dex.runtime_errors import FlowErrorType
 from dex.step import RetryPolicy, StepDurability
 from dex.step_execution import StepExecutionId, TimerId
+from dex.stream import Stream, StreamMessage
 
 InputT = TypeVar("InputT")
 OutputT = TypeVar("OutputT")
@@ -97,6 +98,7 @@ class Client:
             registry,
             self._values,
             self._hydrator,
+            self._service.WriteStream,
         )
         self._closed = False
 
@@ -503,6 +505,109 @@ class Client:
             "active",
         )
 
+    def write_stream(
+        self,
+        flow_id: str,
+        stream: Stream[ValueT],
+        idempotency_key: str,
+        value: ValueT,
+    ) -> None:
+        """Append one typed best-effort Stream message with client idempotency.
+
+        Args:
+            flow_id: Logical Flow instance ID; the Flow need not exist or be active.
+            stream: Exact Stream object registered in one Flow schema.
+            idempotency_key: Non-empty key without ``#``; retained duplicates are no-ops.
+            value: Typed message to append.
+
+        Raises:
+            ValueError: If an ID is empty or the key contains the reserved separator.
+            FlowDefinitionError: If the Stream is not registered.
+            ValueMappingError: If the message cannot be encoded.
+            DexServiceError: If FlowService cannot append the message.
+        """
+        require_name(idempotency_key)
+        if "#" in idempotency_key:
+            raise ValueError("Stream client idempotency key must not contain #")
+        flow = self.registry._flow_for_stream(stream)
+        self._call(
+            self._service.WriteStream,
+            pb.WriteStreamRequest(
+                flow_id=require_name(flow_id),
+                flow_type=flow.name,
+                stream_name=stream.name,
+                max_estimated_bytes=stream.max_estimated_bytes,
+                value=self._values.encode(
+                    value,
+                    self._values.codec(stream.value_type),
+                ),
+                idempotency_key=idempotency_key,
+            ),
+            "write_stream",
+            flow_id,
+            "none",
+        )
+
+    def read_stream(
+        self,
+        flow_id: str,
+        stream: Stream[ValueT],
+        resume_token: str = "",
+        timeout: timedelta | None = None,
+    ) -> StreamMessage[ValueT]:
+        """Block for the next retained Stream message after a resume token.
+
+        Args:
+            flow_id: Logical Flow instance ID used as the Stream instance key.
+            stream: Exact Stream object registered in one Flow schema.
+            resume_token: Previous message token, or empty for the retained head.
+            timeout: Optional non-negative server-side long-poll duration.
+
+        Returns:
+            The decoded message, next resume token, creation time, and idempotency key.
+
+        Raises:
+            LongPollTimeoutError: If no message arrives before the server wait expires.
+            FlowDefinitionError: If the Stream is not registered.
+            ValueMappingError: If the retained message cannot be decoded.
+            DexServiceError: If FlowService cannot perform the read.
+        """
+        flow = self.registry._flow_for_stream(stream)
+        response = cast(
+            pb.ReadStreamResponse,
+            self._call(
+                self._service.ReadStream,
+                pb.ReadStreamRequest(
+                    flow_id=require_name(flow_id),
+                    flow_type=flow.name,
+                    stream_name=stream.name,
+                    resume_token=resume_token,
+                    wait_time_seconds=(
+                        0 if timeout is None else self._seconds32(timeout)
+                    ),
+                ),
+                "read_stream",
+                flow_id,
+                "none",
+            ),
+        )
+        if (
+            not response.HasField("message")
+            or not response.message.HasField("value")
+            or not response.message.HasField("created_time")
+            or not response.message.resume_token
+        ):
+            raise ValueError("Dex returned an incomplete Stream message")
+        return StreamMessage(
+            self._values.decode(
+                response.message.value,
+                self._values.codec(stream.value_type),
+            ),
+            response.message.resume_token,
+            response.message.created_time.ToDatetime(tzinfo=timezone.utc),
+            response.message.idempotency_key,
+        )
+
     def wait_for_flow(
         self,
         flow_id: str,

{dex_python_sdk-0.2.1 → dex_python_sdk-0.2.2}/dex/context.py

@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Protocol, Sequence, TypeVar
 if TYPE_CHECKING:
     from dex.attribute import Attribute, AttributeMap
     from dex.channel import Channel, ChannelMap
+    from dex.stream import Stream
 
 ValueT = TypeVar("ValueT")
 
@@ -189,3 +190,9 @@ class Context(Protocol):
         self,
         definition: ChannelMap[object],
     ) -> tuple[str, ...]: ...
+
+    def _write_stream(
+        self,
+        definition: Stream[ValueT],
+        value: ValueT,
+    ) -> object: ...