pyfect-dev 0.1.0__py3-none-any.whl

pyfect/workers/worker.py

@@ -0,0 +1,273 @@
+"""The parent side of workers (Effect's ``Worker``).
+
+A ``Worker[O, I]`` sends ``I`` messages to a worker and runs a handler for
+each ``O`` it sends back. ``WorkerPlatform`` spawns workers by numeric id,
+finding the platform's worker for each id with the ``Spawner`` function in
+the context (``layer_spawner``). ``make_platform(setup=, listen=)`` builds a
+platform from a transport's two hooks: ``setup`` starts the worker and
+returns its port, ``listen`` hands what the worker sends to ``emit`` and
+fails the run through ``deferred`` when the worker goes away. Messages sent
+before the worker says it is ready are kept and sent once it is.
+
+``python_worker.layer`` is the platform for Python threads and child
+processes (Effect's ``NodeWorker``).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from typing import Any, Literal, Never, Protocol, cast
+
+from pyfect._core import Effect, Restore, uninterruptible_mask
+from pyfect._effect_args import require_effect
+from pyfect.context import Tag
+from pyfect.deferred import Deferred
+from pyfect.effect import gen, provide_service, succeed, suspend, sync, try_
+from pyfect.fiber_set import FiberSet
+from pyfect.latch import Latch
+from pyfect.layer import Layer
+from pyfect.layer import succeed as layer_succeed
+from pyfect.scope import CloseableScope, Scope, scoped_with
+from pyfect.workers.worker_error import WorkerError, WorkerSendError
+
+type PlatformMessage = tuple[Literal[0]] | tuple[Literal[1], object]
+"""What a worker sends its parent: ``(0,)`` when it is ready, ``(1, data)``
+for a message."""
+
+type Transfers = Sequence[object]
+"""Values a platform may move rather than copy with a message (Effect's
+transfer list). The Python platforms copy or share every message and
+ignore them."""
+
+type SendFn = Callable[[Any, Transfers], Effect[None, WorkerError]]
+type RunFn = Callable[
+    [Callable[[Any], Effect[Any, Any, Any]], Effect[None] | None], Effect[Never, Any, Any]
+]
+
+_VOID: Effect[None] = succeed(None)
+
+
+class Worker[O = Any, I = Any]:
+    """A spawned worker: ``send`` a message, ``run`` a handler for its
+    replies. ``run`` starts the worker and never returns; it fails when
+    the worker goes away or the handler fails."""
+
+    __slots__ = ("_run", "_send")
+
+    def __init__(self, *, send: SendFn, run: RunFn) -> None:
+        self._send = send
+        self._run = run
+
+    def send(self, message: I, transfers: Transfers = ()) -> Effect[None, WorkerError]:
+        return self._send(message, transfers)
+
+    def run[A, E = Never, R = Never](
+        self,
+        handler: Callable[[O], Effect[A, E, R]],
+        *,
+        on_spawn: Effect[None] | None = None,
+    ) -> Effect[Never, E | WorkerError, R]:
+        """Start the worker and handle each of its messages on a fiber of
+        its own; ``on_spawn`` runs once the worker is ready."""
+        if on_spawn is not None:
+            require_effect(on_spawn, "on_spawn")
+        return cast("Effect[Never, E | WorkerError, R]", self._run(handler, on_spawn))
+
+
+def make_unsafe(
+    *,
+    send: SendFn,
+    run: Callable[[Callable[[PlatformMessage], Effect[Any, Any, Any]]], Effect[Never, Any, Any]],
+) -> Worker[Any, Any]:
+    """A ``Worker`` over a platform's raw ``send`` and ``run``: ``run``'s
+    handler gets the ``PlatformMessage``s, and a ready message runs
+    ``on_spawn``."""
+
+    def run_worker(
+        handler: Callable[[Any], Effect[Any, Any, Any]], on_spawn: Effect[None] | None
+    ) -> Effect[Never, Any, Any]:
+        spawned = _VOID if on_spawn is None else on_spawn
+
+        def on_message(message: PlatformMessage) -> Effect[Any, Any, Any]:
+            return spawned if message[0] == 0 else handler(message[1])
+
+        return run(on_message)
+
+    return Worker(send=send, run=run_worker)
+
+
+type SpawnerFn = Callable[[int], object]
+"""Gives the platform's worker for a worker id."""
+
+
+class Spawner(Tag[SpawnerFn], key="effect/workers/Worker/Spawner"):
+    """How the ``WorkerPlatform`` finds the worker for each id."""
+
+
+def layer_spawner(spawner: SpawnerFn) -> Layer[Spawner]:
+    return layer_succeed(Spawner, spawner)
+
+
+@dataclass(frozen=True, slots=True)
+class WorkerPlatformService:
+    """``spawn(id)`` gives the worker for ``id`` (from the ``Spawner``); it
+    starts when it is run."""
+
+    spawn: Callable[[int], Effect[Worker[Any, Any], WorkerError, Spawner]]
+
+
+class WorkerPlatform(Tag[WorkerPlatformService], key="effect/workers/Worker/WorkerPlatform"):
+    """Spawns the workers of the current platform."""
+
+
+class Port(Protocol):
+    """What a platform's ``setup`` gives: where messages to the worker go."""
+
+    def post_message(self, message: object, transfers: Transfers) -> None: ...
+
+
+@dataclass(frozen=True, slots=True)
+class SetupOptions[W]:
+    worker: W
+    scope: CloseableScope
+
+
+@dataclass(frozen=True, slots=True)
+class ListenOptions[P]:
+    """``emit`` each ``PlatformMessage`` the worker sends (on the runtime's
+    thread); complete ``deferred`` with a ``WorkerError`` when it goes
+    away. ``scope`` closes when the run ends."""
+
+    port: P
+    emit: Callable[[PlatformMessage], None]
+    deferred: Deferred[None, WorkerError]
+    scope: CloseableScope
+
+
+class _Spawned[P: Port]:
+    __slots__ = ("buffer", "listen", "port", "setup", "worker")
+
+    def __init__(
+        self,
+        worker: Callable[[], object],
+        setup: Callable[[SetupOptions[Any]], Effect[P, WorkerError]],
+        listen: Callable[[ListenOptions[P]], Effect[None]],
+    ) -> None:
+        self.worker = worker
+        self.setup = setup
+        self.listen = listen
+        self.port: P | None = None
+        self.buffer: list[tuple[object, Transfers]] = []
+
+    def send_to_port(
+        self, port: P, message: object, transfers: Transfers
+    ) -> Effect[None, WorkerError]:
+        def post() -> None:
+            port.post_message((0, message), transfers)
+
+        return try_(
+            post,
+            lambda cause: WorkerError(
+                reason=WorkerSendError(message="Failed to send message to worker", cause=cause)
+            ),
+        )
+
+    def send(self, message: object, transfers: Transfers) -> Effect[None, WorkerError]:
+        def body() -> Effect[None, WorkerError]:
+            if self.port is None:
+                self.buffer.append((message, transfers))
+                return _VOID
+            return self.send_to_port(self.port, message, transfers)
+
+        return suspend(body)
+
+    def run(
+        self, handler: Callable[[Any], Effect[Any, Any, Any]], on_spawn: Effect[None] | None
+    ) -> Effect[Never, Any, Any]:
+        return uninterruptible_mask(
+            lambda restore: scoped_with(
+                lambda scope: self._running(handler, on_spawn, restore, scope)
+            )
+        )
+
+    def _forget_port(self) -> None:
+        self.port = None
+
+    def _running(
+        self,
+        handler: Callable[[Any], Effect[Any, Any, Any]],
+        on_spawn: Effect[None] | None,
+        restore: Restore,
+        scope: CloseableScope,
+    ) -> Effect[Never, Any, Any]:
+        @gen
+        def body():
+            port = yield from self.setup(SetupOptions(self.worker(), scope))
+            yield from scope.add_finalizer(lambda _: sync(self._forget_port))
+            fiber_set = yield from FiberSet[Any, Any].make().pipe(provide_service(Scope, scope))
+            run = yield from fiber_set.runtime()
+            ready = Latch(False)
+
+            def emit(data: PlatformMessage) -> None:
+                if data[0] == 0:
+                    if on_spawn is None:
+                        ready.open_unsafe()
+                    else:
+                        run(on_spawn.ensuring(ready.open()))
+                    return
+                run(handler(data[1]))
+
+            yield from self.listen(ListenOptions(port, emit, fiber_set.deferred, scope))
+            # Fail fast if the worker goes away before it is ready, and stay
+            # interruptible while waiting for it.
+            yield from restore(ready.wait().race_first(fiber_set.join()))
+            self.port = port
+            pending, self.buffer = self.buffer, []
+            for message, transfers in pending:
+                yield from self.send_to_port(port, message, transfers)
+            yield from restore(fiber_set.join())
+            return cast("Never", None)
+
+        return body
+
+
+def make_platform[W, P: Port](
+    *,
+    setup: Callable[[SetupOptions[W]], Effect[P, WorkerError]],
+    listen: Callable[[ListenOptions[P]], Effect[None]],
+) -> WorkerPlatformService:
+    """A ``WorkerPlatform`` from a transport's hooks (Effect's
+    ``makePlatform``). ``setup`` starts the worker the ``Spawner`` gave and
+    returns its port, adding to ``scope`` whatever stops it (each run asks
+    the ``Spawner`` for a new worker); ``listen``
+    wires the port's messages and failures."""
+
+    def spawn(worker_id: int) -> Effect[Worker[Any, Any], WorkerError, Spawner]:
+        def made(spawner: SpawnerFn) -> Worker[Any, Any]:
+            spawned = _Spawned[P](lambda: spawner(worker_id), cast("Any", setup), listen)
+            return Worker(send=spawned.send, run=spawned.run)
+
+        return Spawner().map(made)
+
+    return WorkerPlatformService(spawn)
+
+
+__all__ = [
+    "ListenOptions",
+    "PlatformMessage",
+    "Port",
+    "RunFn",
+    "SendFn",
+    "SetupOptions",
+    "Spawner",
+    "SpawnerFn",
+    "Transfers",
+    "Worker",
+    "WorkerError",
+    "WorkerPlatform",
+    "WorkerPlatformService",
+    "layer_spawner",
+    "make_platform",
+    "make_unsafe",
+]

pyfect/workers/worker_error.py

@@ -0,0 +1,86 @@
+"""The errors of workers (Effect's ``WorkerError``): a ``WorkerError``
+wrapping why, a spawn, send, receive or unknown failure."""
+
+from __future__ import annotations
+
+from typing import TypeIs
+
+from pyfect import schema as S
+
+
+def _cause_field() -> object:
+    return S.field(S.Union(S.Null, S.Defect()), default=None)
+
+
+class WorkerSpawnError(S.TaggedError, identifier="effect/workers/WorkerError/WorkerSpawnError"):
+    """A worker could not be spawned or set up."""
+
+    message: str
+    cause: object = _cause_field()
+
+
+class WorkerSendError(S.TaggedError, identifier="effect/workers/WorkerError/WorkerSendError"):
+    """A message could not be sent to a worker."""
+
+    message: str
+    cause: object = _cause_field()
+
+
+class WorkerReceiveError(S.TaggedError, identifier="effect/workers/WorkerError/WorkerReceiveError"):
+    """A message from a worker could not be received, or the worker went
+    away (it exited, or raised)."""
+
+    message: str
+    cause: object = _cause_field()
+
+
+class WorkerUnknownError(S.TaggedError, identifier="effect/workers/WorkerError/WorkerUnknownError"):
+    """A worker failed in a way none of the other reasons cover."""
+
+    message: str
+    cause: object = _cause_field()
+
+
+type WorkerErrorReason = (
+    WorkerSpawnError | WorkerSendError | WorkerReceiveError | WorkerUnknownError
+)
+
+WORKER_ERROR_REASON = S.Union(
+    WorkerSpawnError, WorkerSendError, WorkerReceiveError, WorkerUnknownError
+)
+"""The schema of every reason (Effect's ``WorkerErrorReason`` schema)."""
+
+
+class WorkerError(S.TaggedError, identifier="~effect/workers/WorkerError"):
+    """A worker failure; ``reason`` says which kind. Its message is the
+    reason's, and its ``cause`` (and ``__cause__``) the reason's cause."""
+
+    reason: WorkerErrorReason = S.field(WORKER_ERROR_REASON)
+
+    def __post_init__(self) -> None:
+        self.__cause__ = self.cause if isinstance(self.cause, BaseException) else None
+
+    @property
+    def cause(self) -> object:
+        return self.reason.cause
+
+    @property
+    def message(self) -> str:
+        return self.reason.message
+
+
+def is_worker_error(value: object) -> TypeIs[WorkerError]:
+    """Whether ``value`` is a ``WorkerError`` (not one of its reasons)."""
+    return isinstance(value, WorkerError)
+
+
+__all__ = [
+    "WORKER_ERROR_REASON",
+    "WorkerError",
+    "WorkerErrorReason",
+    "WorkerReceiveError",
+    "WorkerSendError",
+    "WorkerSpawnError",
+    "WorkerUnknownError",
+    "is_worker_error",
+]

pyfect/workers/worker_runner.py

@@ -0,0 +1,85 @@
+"""The worker side of workers (Effect's ``WorkerRunner``).
+
+A ``WorkerRunner[O, I]`` runs inside a worker: ``run(handler)`` tells the
+parent the worker is ready, then calls ``handler(port_id, message)`` for
+each ``I`` the parent sends, running the effect it returns (if any) on a
+fiber of its own, until the parent closes the worker. ``send`` replies with
+an ``O``. ``WorkerRunnerPlatform`` starts the runner of the current
+platform; ``python_worker_runner.layer`` is the one for Python threads and
+child processes (Effect's ``NodeWorkerRunner``).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any, Literal, Never, cast
+
+from pyfect._core import Effect
+from pyfect.context import Tag
+from pyfect.effect import sync
+from pyfect.queue import Dequeue
+from pyfect.workers.worker import Transfers
+from pyfect.workers.worker_error import WorkerError
+
+type PlatformMessage[I] = tuple[Literal[0], I] | tuple[Literal[1]]
+"""What a parent sends its worker: ``(0, message)``, or ``(1,)`` to close
+it."""
+
+type Handler = Callable[[int, Any], Effect[Any, Any, Any] | None]
+
+
+class WorkerRunner[O = Any, I = Any]:
+    """The worker's end: ``run`` a handler for the parent's messages,
+    ``send`` it replies. ``disconnects`` gives the ports that went away, on
+    platforms with several (``None`` otherwise)."""
+
+    __slots__ = ("_run", "_send_unsafe", "disconnects")
+
+    def __init__(
+        self,
+        *,
+        run: Callable[[Handler], Effect[None, WorkerError, Any]],
+        send_unsafe: Callable[[int, Any, Transfers], None],
+        disconnects: Dequeue[int, Never] | None = None,
+    ) -> None:
+        self._run = run
+        self._send_unsafe = send_unsafe
+        self.disconnects = disconnects
+
+    def run[A, E = Never, R = Never](
+        self, handler: Callable[[int, I], Effect[A, E, R] | None]
+    ) -> Effect[None, WorkerError, R]:
+        """Handle the parent's messages until it closes the worker. A
+        handler's failure is logged, not propagated."""
+        return cast("Effect[None, WorkerError, R]", self._run(handler))
+
+    def send(self, port_id: int, message: O, transfers: Transfers = ()) -> Effect[None]:
+        return sync(lambda: self._send_unsafe(port_id, message, transfers))
+
+    def send_unsafe(self, port_id: int, message: O, transfers: Transfers = ()) -> None:
+        self._send_unsafe(port_id, message, transfers)
+
+
+@dataclass(frozen=True, slots=True)
+class WorkerRunnerPlatformService:
+    """``start()`` gives the runner of the worker this code runs in; it
+    fails with a ``WorkerError`` outside a worker."""
+
+    start: Callable[[], Effect[WorkerRunner[Any, Any], WorkerError]]
+
+
+class WorkerRunnerPlatform(
+    Tag[WorkerRunnerPlatformService], key="effect/workers/WorkerRunner/WorkerRunnerPlatform"
+):
+    """Starts the runner of the current platform's worker."""
+
+
+__all__ = [
+    "Handler",
+    "PlatformMessage",
+    "WorkerError",
+    "WorkerRunner",
+    "WorkerRunnerPlatform",
+    "WorkerRunnerPlatformService",
+]

pyfect/workflow/__init__.py

@@ -0,0 +1,78 @@
+"""Durable workflows (Effect's ``effect/workflow``)::
+
+    from pyfect.workflow import activity, durable_deferred, workflow, workflow_engine
+
+    Order = workflow.make(
+        "Order",
+        payload=S.Struct(OrderPayload),
+        success=S.String,
+        idempotency_key=lambda order: order["id"],
+    )
+
+    def run_order(order: OrderPayload, execution_id: str):
+        charge_id = yield from activity.make(
+            name="charge", success=S.String, execute=charge(order)
+        ).pipe(Order.with_compensation(lambda charge_id, cause: refund(charge_id)))
+        yield from durable_clock.sleep(name="cool-off", duration=3600)
+        return charge_id
+
+    OrderLive = Order.to_layer(fx.fn(run_order)).pipe(
+        fx.layer.provide_merge(workflow_engine.layer_memory)
+    )
+
+- ``workflow``: ``Workflow`` (``make``, ``execute``, ``poll``,
+  ``interrupt``, ``resume``, ``to_layer``, ``with_compensation``), its
+  results ``Complete`` / ``Suspended``, ``CaptureDefects`` and
+  ``SuspendOnFailure``, the execution's ``scope`` and ``add_finalizer``.
+- ``activity``: steps whose results are stored and replayed.
+- ``durable_deferred``: named wait points completed from outside by token.
+- ``durable_clock``: durable sleeps.
+- ``durable_queue``: work handed to ``PersistedQueue`` workers.
+- ``workflow_engine``: the ``WorkflowEngine`` service and ``layer_memory``.
+- ``workflow_proxy`` / ``workflow_proxy_server``: RPC and HTTP API groups
+  that execute, discard and resume workflows. Import them by name
+  (``from pyfect.workflow import workflow_proxy``); they load the RPC and
+  HTTP API packages, which ``import pyfect`` leaves out.
+"""
+
+from pyfect.workflow import (
+    activity,
+    durable_clock,
+    durable_deferred,
+    durable_queue,
+    workflow,
+    workflow_engine,
+)
+from pyfect.workflow._result import Complete, Result, Suspended
+from pyfect.workflow._services import (
+    WorkflowEngine,
+    WorkflowEngineService,
+    WorkflowInstance,
+    WorkflowInstanceService,
+)
+from pyfect.workflow.activity import Activity
+from pyfect.workflow.durable_clock import DurableClock
+from pyfect.workflow.durable_deferred import DurableDeferred
+from pyfect.workflow.durable_queue import DurableQueue
+from pyfect.workflow.workflow import Workflow
+
+__all__ = [
+    "Activity",
+    "Complete",
+    "DurableClock",
+    "DurableDeferred",
+    "DurableQueue",
+    "Result",
+    "Suspended",
+    "Workflow",
+    "WorkflowEngine",
+    "WorkflowEngineService",
+    "WorkflowInstance",
+    "WorkflowInstanceService",
+    "activity",
+    "durable_clock",
+    "durable_deferred",
+    "durable_queue",
+    "workflow",
+    "workflow_engine",
+]

pyfect/workflow/_crypto.py

@@ -0,0 +1,23 @@
+"""Execution ids and idempotency keys: the first 16 bytes of a SHA-256
+digest, in hex (Effect's ``internal/crypto.ts``)."""
+
+from __future__ import annotations
+
+import hashlib
+
+from pyfect._core import Effect
+from pyfect._js import text_encode
+from pyfect.effect import sync, yield_now
+
+
+def make_hash_digest(original: str) -> Effect[str]:
+    """The digest of ``original``'s UTF-8 bytes. Like Effect's, which
+    awaits ``crypto.subtle.digest``, it lets other fibers run first."""
+
+    def digest() -> str:
+        return hashlib.sha256(text_encode(original)).digest()[:16].hex()
+
+    return yield_now().and_then(sync(digest))
+
+
+__all__ = ["make_hash_digest"]