pyfect-dev 0.1.0__py3-none-any.whl

pyfect/tx_ref.py

@@ -0,0 +1,79 @@
+"""``TxRef[A]``: a mutable cell read and written in transactions.
+
+Every operation joins the running transaction (``fx.tx``), or runs as a
+transaction of its own when there is none. Inside a transaction, reads and
+writes go to its journal; other fibers see the writes only once the
+outermost transaction commits.
+"""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import overload
+
+from pyfect._core import Effect, _Sync
+from pyfect._unsubscripted import Unsubscripted
+from pyfect.transaction import TransactionMeta, transactional
+
+
+class TxRef[A = Unsubscripted]:
+    """``TxRef(value)`` is Effect's ``makeUnsafe``; ``TxRef.make`` is the
+    effect. A committed write that changes the value (by JavaScript's
+    ``Object.is``: ``-0.0`` replaces ``0.0``) bumps ``version``; every
+    committed write, changed or not, wakes the transactions waiting in
+    ``tx_retry`` on this ref."""
+
+    __slots__ = ("_pending", "_value", "_version")
+
+    def __init__(self, value: A) -> None:
+        self._value = value
+        self._version = 0
+        # Wake callbacks of transactions parked in ``tx_retry`` after reading
+        # this ref, keyed by waiter.
+        self._pending: dict[object, Callable[[], None]] = {}
+
+    @overload
+    @classmethod
+    def make[T](cls: type[TxRef[Unsubscripted]], value: T) -> Effect[TxRef[T]]: ...
+    @overload
+    @classmethod
+    def make(cls, value: A) -> Effect[TxRef[A]]: ...
+    @classmethod
+    def make[T](cls, value: T) -> Effect[TxRef[T]]:
+        return _Sync(lambda: TxRef(value))
+
+    @property
+    def version(self) -> int:
+        """How many committed writes changed the value."""
+        return self._version
+
+    def get(self) -> Effect[A]:
+        return transactional(self._get)
+
+    def set(self, value: A) -> Effect[None]:
+        return self.modify(lambda _: (None, value))
+
+    def update(self, f: Callable[[A], A]) -> Effect[None]:
+        return self.modify(lambda current: (None, f(current)))
+
+    def modify[B](self, f: Callable[[A], tuple[B, A]]) -> Effect[B]:
+        """Replace the value with the second item of ``f(value)``; succeed
+        with the first."""
+
+        def run(state: TransactionMeta) -> B:
+            entry = state._entry(self)
+            result, entry._value = f(entry._value)
+            entry._written = True
+            return result
+
+        return transactional(run)
+
+    def _get(self, state: TransactionMeta) -> A:
+        return state._entry(self)._value
+
+    def __repr__(self) -> str:
+        return f"TxRef({self._value!r})"
+
+
+__all__ = ["TxRef"]

pyfect/tx_semaphore.py

@@ -0,0 +1,136 @@
+"""``TxSemaphore``: a transactional counting semaphore (Effect's
+``TxSemaphore``).
+
+The free permits live in a ``TxRef``. ``acquire`` waits (``tx_retry``)
+until enough permits are free; ``release`` never raises the count above the
+capacity it was made with. Inside a transaction, acquiring and releasing
+take effect only when it commits.
+"""
+
+from __future__ import annotations
+
+from typing import Never
+
+from pyfect import _tx_ops
+from pyfect._core import Effect, Restore, uninterruptible_mask
+from pyfect._effect_args import require_effect
+from pyfect.effect import die, void
+from pyfect.scope import Scope, add_finalizer
+from pyfect.transaction import Transaction, TransactionMeta, tx_restoring, tx_retry
+from pyfect.tx_ref import TxRef
+
+
+def _positive(n: int) -> Effect[None]:
+    """Effect dies, rather than raising, on a count that is not positive."""
+    return void if n > 0 else die(ValueError("Number of permits must be positive"))
+
+
+class TxSemaphore:
+    """``TxSemaphore(ref, capacity)`` wraps an existing ``TxRef`` (Effect's
+    ``makeUnsafe``); ``make`` is the effect."""
+
+    __slots__ = ("_capacity", "_permits")
+
+    def __init__(self, permits: TxRef[int], capacity: int) -> None:
+        self._permits = permits
+        self._capacity = capacity
+
+    @staticmethod
+    def make(permits: int) -> Effect[TxSemaphore]:
+        """A semaphore with ``permits`` free permits, also its capacity. Dies
+        if ``permits`` is negative."""
+        if permits < 0:
+            return die(ValueError("Permits must be non-negative"))
+        return TxRef.make(permits).map(lambda ref: TxSemaphore(ref, permits))
+
+    @property
+    def capacity(self) -> int:
+        return self._capacity
+
+    @property
+    def ref(self) -> TxRef[int]:
+        """The free permits."""
+        return self._permits
+
+    def available(self) -> Effect[int]:
+        return self._permits.get()
+
+    def acquire(self) -> Effect[None]:
+        """Take one permit, waiting until one is free."""
+        return self._acquire(1)
+
+    def acquire_n(self, n: int) -> Effect[None]:
+        """Take ``n`` permits at once, waiting until that many are free.
+        Dies if ``n`` is not positive."""
+        return _positive(n).and_then(self._acquire(n))
+
+    def try_acquire(self) -> Effect[bool]:
+        """Take one permit if one is free now."""
+        return self._try_acquire(1)
+
+    def try_acquire_n(self, n: int) -> Effect[bool]:
+        """Take ``n`` permits if that many are free now. Dies if ``n`` is not
+        positive."""
+        return _positive(n).and_then(self._try_acquire(n))
+
+    def release(self) -> Effect[None]:
+        """Give back one permit, up to the capacity."""
+        return self._release(1)
+
+    def release_n(self, n: int) -> Effect[None]:
+        """Give back ``n`` permits, up to the capacity. Dies if ``n`` is not
+        positive."""
+        return _positive(n).and_then(self._release(n))
+
+    def with_permit[A, E, R](self, eff: Effect[A, E, R]) -> Effect[A, E, R]:
+        """Run ``eff`` holding one permit, giving it back however it ends.
+        Only the wait for the permit is interruptible."""
+        require_effect(eff, "eff")
+        return self._with_acquired(self.acquire(), eff, self.release())
+
+    def with_permits[A, E, R](self, eff: Effect[A, E, R], n: int) -> Effect[A, E, R]:
+        """``with_permit`` for ``n`` permits."""
+        require_effect(eff, "eff")
+        return self._with_acquired(self.acquire_n(n), eff, self.release_n(n))
+
+    def with_permit_scoped(self) -> Effect[None, Never, Scope]:
+        """Take one permit until the current scope closes. Only the wait for
+        the permit is interruptible."""
+        release = self.release()
+        return uninterruptible_mask(
+            lambda restore: tx_restoring(restore, self.acquire()).and_then(
+                add_finalizer(lambda _: release)
+            )
+        )
+
+    def _acquire(self, n: int) -> Effect[None]:
+        def take(state: TransactionMeta) -> Effect[None, Never, Transaction]:
+            permits = _tx_ops.value(state, self._permits)
+            if permits < n:
+                return tx_retry
+            _tx_ops.write(state, self._permits, permits - n)
+            return void
+
+        return _tx_ops.decide(take)
+
+    def _try_acquire(self, n: int) -> Effect[bool]:
+        return self._permits.modify(
+            lambda permits: (True, permits - n) if permits >= n else (False, permits)
+        )
+
+    def _release(self, n: int) -> Effect[None]:
+        return self._permits.update(lambda permits: min(permits + n, self._capacity))
+
+    def _with_acquired[A, E, R](
+        self, acquire: Effect[None], eff: Effect[A, E, R], release: Effect[None]
+    ) -> Effect[A, E, R]:
+        def body(restore: Restore) -> Effect[A, E, R]:
+            return tx_restoring(restore, acquire).and_then(restore(eff).ensuring(release))
+
+        return uninterruptible_mask(body)
+
+    def __repr__(self) -> str:
+        return f"TxSemaphore(capacity={self._capacity})"
+
+
+__all__ = ["TxSemaphore"]

pyfect/tx_subscription_ref.py

@@ -0,0 +1,118 @@
+"""``TxSubscriptionRef[A]``: a ``TxRef`` that publishes every value it
+stores to an unbounded ``TxPubSub`` (Effect's ``TxSubscriptionRef``).
+
+``changes()`` subscribes for the current scope: the queue's first value is
+the current one, then every value stored after it, in commit order.
+``changes_stream()`` is the same as a ``Stream``, subscribing when it
+starts. A write publishes in the same transaction, so a transaction that
+rolls back publishes nothing.
+"""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Never, overload
+
+from pyfect._core import Effect
+from pyfect._unsubscripted import Unsubscripted
+from pyfect.effect import gen
+from pyfect.scope import Scope, acquire_release
+from pyfect.transaction import tx
+from pyfect.tx_pubsub import TxPubSub
+from pyfect.tx_queue import TxDequeue
+from pyfect.tx_ref import TxRef
+
+if TYPE_CHECKING:
+    from pyfect.stream import Stream
+
+
+class TxSubscriptionRef[A = Unsubscripted]:
+    """``make`` is the effect; ``isinstance(x, TxSubscriptionRef)`` is
+    Effect's ``isTxSubscriptionRef``."""
+
+    __slots__ = ("_pubsub", "_ref")
+
+    def __init__(self, ref: TxRef[A], pubsub: TxPubSub[A]) -> None:
+        self._ref = ref
+        self._pubsub = pubsub
+
+    @overload
+    @classmethod
+    def make[T](
+        cls: type[TxSubscriptionRef[Unsubscripted]], value: T
+    ) -> Effect[TxSubscriptionRef[T]]: ...
+    @overload
+    @classmethod
+    def make(cls, value: A) -> Effect[TxSubscriptionRef[A]]: ...
+    @classmethod
+    def make[T](cls, value: T) -> Effect[TxSubscriptionRef[T]]:
+        return TxPubSub[T].unbounded().map(lambda pubsub: TxSubscriptionRef(TxRef(value), pubsub))
+
+    @property
+    def ref(self) -> TxRef[A]:
+        return self._ref
+
+    @property
+    def pubsub(self) -> TxPubSub[A]:
+        return self._pubsub
+
+    def get(self) -> Effect[A]:
+        return self._ref.get()
+
+    def modify[B](self, f: Callable[[A], tuple[B, A]]) -> Effect[B]:
+        """Store the second item of ``f(value)`` and publish it; succeed with
+        the first."""
+
+        @gen
+        def body():
+            result, value = f((yield from self._ref.get()))
+            yield from self._ref.set(value)
+            yield from self._pubsub.publish(value)
+            return result
+
+        return tx(body)
+
+    def set(self, value: A) -> Effect[None]:
+        return self.modify(lambda _: (None, value))
+
+    def update(self, f: Callable[[A], A]) -> Effect[None]:
+        return self.modify(lambda current: (None, f(current)))
+
+    def get_and_set(self, value: A) -> Effect[A]:
+        return self.modify(lambda current: (current, value))
+
+    def get_and_update(self, f: Callable[[A], A]) -> Effect[A]:
+        return self.modify(lambda current: (current, f(current)))
+
+    def update_and_get(self, f: Callable[[A], A]) -> Effect[A]:
+        def step(current: A) -> tuple[A, A]:
+            new = f(current)
+            return new, new
+
+        return self.modify(step)
+
+    def changes(self) -> Effect[TxDequeue[A, Never], Never, Scope]:
+        """A queue of the current value, then every later one, until the
+        current scope closes."""
+
+        @gen
+        def acquire():
+            queue = yield from self._pubsub._acquire_queue()
+            yield from queue.offer((yield from self._ref.get()))
+            return queue
+
+        return acquire_release(tx(acquire), lambda queue, _: self._pubsub.release_subscriber(queue))
+
+    def changes_stream(self) -> Stream[A]:
+        """A stream of the current value, then every later one (Effect's
+        ``changesStream``). Each run subscribes anew."""
+        from pyfect.stream import from_effect_repeat, unwrap
+
+        return unwrap(self.changes().map(lambda queue: from_effect_repeat(queue.take())))
+
+    def __repr__(self) -> str:
+        return "TxSubscriptionRef()"
+
+
+__all__ = ["TxSubscriptionRef"]

pyfect/workers/__init__.py

@@ -0,0 +1,62 @@
+"""Workers (Effect's ``effect/workers``): run code on other threads or in
+other processes and talk to it with messages.
+
+- ``worker``: the parent side, ``Worker``, ``WorkerPlatform`` and
+  ``Spawner``, and ``make_platform`` for transports.
+- ``worker_runner``: the worker side, ``WorkerRunner`` and
+  ``WorkerRunnerPlatform``.
+- ``worker_error``: ``WorkerError`` and its reasons.
+- ``transferable``: ``Collector`` and transferable schemas.
+- ``python_worker`` / ``python_worker_runner``: the platform for Python
+  threads (``ThreadWorker``) and child processes (``ProcessWorker``),
+  Effect's ``NodeWorker`` / ``NodeWorkerRunner``.
+
+``pyfect.rpc`` serves and calls RPC groups over workers
+(``rpc.client.layer_protocol_worker``,
+``rpc.server.layer_protocol_worker_runner``).
+"""
+
+from pyfect.workers import (
+    python_worker,
+    python_worker_runner,
+    transferable,
+    worker,
+    worker_error,
+    worker_runner,
+)
+from pyfect.workers.python_worker import ProcessWorker, ThreadWorker, WorkerHandle
+from pyfect.workers.worker import Spawner, Worker, WorkerPlatform, layer_spawner, make_platform
+from pyfect.workers.worker_error import (
+    WorkerError,
+    WorkerReceiveError,
+    WorkerSendError,
+    WorkerSpawnError,
+    WorkerUnknownError,
+    is_worker_error,
+)
+from pyfect.workers.worker_runner import WorkerRunner, WorkerRunnerPlatform
+
+__all__ = [
+    "ProcessWorker",
+    "Spawner",
+    "ThreadWorker",
+    "Worker",
+    "WorkerError",
+    "WorkerHandle",
+    "WorkerPlatform",
+    "WorkerReceiveError",
+    "WorkerRunner",
+    "WorkerRunnerPlatform",
+    "WorkerSendError",
+    "WorkerSpawnError",
+    "WorkerUnknownError",
+    "is_worker_error",
+    "layer_spawner",
+    "make_platform",
+    "python_worker",
+    "python_worker_runner",
+    "transferable",
+    "worker",
+    "worker_error",
+    "worker_runner",
+]

pyfect/workers/_frames.py

@@ -0,0 +1,82 @@
+"""Messages over a socket between a parent and its worker process.
+
+Each message is a pickle behind its four-byte length. A ``FrameWriter``
+writes on a thread of its own, so posting never waits on the socket;
+``read_frames`` reads on the calling thread (a reader thread) until the
+other end closes.
+"""
+
+from __future__ import annotations
+
+import pickle
+import queue
+import socket
+import struct
+import threading
+from collections.abc import Callable
+
+_HEADER = struct.Struct(">I")
+
+
+def encode(message: object) -> bytes:
+    payload = pickle.dumps(message, protocol=pickle.HIGHEST_PROTOCOL)
+    return _HEADER.pack(len(payload)) + payload
+
+
+def read_frames(
+    sock: socket.socket,
+    on_message: Callable[[object], None],
+    on_unreadable: Callable[[BaseException], None],
+) -> None:
+    """Read messages until the socket ends or fails."""
+    with sock.makefile("rb") as stream:
+        while True:
+            try:
+                header = stream.read(_HEADER.size)
+                if len(header) < _HEADER.size:
+                    return
+                (size,) = _HEADER.unpack(header)
+                payload = stream.read(size)
+            except OSError:
+                return
+            if len(payload) < size:
+                return
+            try:
+                message = pickle.loads(payload)  # our own child or parent
+            except Exception as cause:  # reported as an unreadable message
+                on_unreadable(cause)
+                continue
+            on_message(message)
+
+
+class FrameWriter:
+    """Writes frames in order on its own thread; once the socket fails,
+    frames are dropped."""
+
+    __slots__ = ("_frames", "_sock", "_thread")
+
+    def __init__(self, sock: socket.socket, name: str) -> None:
+        self._sock = sock
+        self._frames: queue.SimpleQueue[bytes | None] = queue.SimpleQueue()
+        self._thread = threading.Thread(target=self._run, name=name, daemon=True)
+        self._thread.start()
+
+    def write(self, frame: bytes) -> None:
+        self._frames.put(frame)
+
+    def close(self, timeout: float | None = None) -> None:
+        """Stop once what was written is sent, waiting at most ``timeout``
+        for it."""
+        self._frames.put(None)
+        if timeout is not None and threading.current_thread() is not self._thread:
+            self._thread.join(timeout)
+
+    def _run(self) -> None:
+        while True:
+            frame = self._frames.get()
+            if frame is None:
+                return
+            try:
+                self._sock.sendall(frame)
+            except OSError:
+                return

pyfect/workers/_mailbox.py

@@ -0,0 +1,51 @@
+"""A ``Mailbox``: items put from any thread, taken in order by a fiber.
+
+The threads of a worker's transport (a reader, another runtime's
+scheduler) put what arrives; one fiber takes it, so everything a worker
+receives is handled on its runtime's own thread. A waiting take suspends
+in ``os_wait``; a put wakes it to retry, never handing it the item.
+"""
+
+from __future__ import annotations
+
+import threading
+from collections import deque
+from typing import Never
+
+from pyfect._core import Effect
+from pyfect._os_wait import Cancel, Resume, os_wait
+from pyfect.effect import succeed, suspend
+
+
+class Mailbox[A]:
+    __slots__ = ("_items", "_lock", "_waiters")
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._items: deque[A] = deque()
+        self._waiters: list[Resume[A, Never]] = []
+
+    def put(self, item: A) -> None:
+        with self._lock:
+            self._items.append(item)
+            waiters, self._waiters = self._waiters, []
+        for resume in waiters:
+            resume(suspend(self.take))
+
+    def take(self) -> Effect[A]:
+        def register(resume: Resume[A, Never]) -> Cancel | None:
+            with self._lock:
+                taken = (self._items.popleft(),) if self._items else None
+                if taken is None:
+                    self._waiters.append(resume)
+            if taken is not None:
+                resume(succeed(taken[0]))
+                return None
+            return lambda: self._forget(resume)
+
+        return os_wait(register, waits_for="event")
+
+    def _forget(self, resume: Resume[A, Never]) -> None:
+        with self._lock:
+            if resume in self._waiters:
+                self._waiters.remove(resume)

pyfect/workers/_port.py

@@ -0,0 +1,106 @@
+"""The two ends of a Python worker's transport, as the platforms see them.
+
+The parent holds a ``WorkerPort``: it posts messages to the worker, and the
+worker's messages, receive errors and exit arrive in ``events``. The worker
+holds a ``ParentPort``: it posts to the parent, and the parent's messages
+arrive in ``inbox``. Each backend (a thread, a child process) implements
+both; the threads moving messages only ever put into a ``Mailbox``.
+"""
+
+from __future__ import annotations
+
+import threading
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+from pyfect.process._os import Latch
+from pyfect.workers._mailbox import Mailbox
+
+
+@dataclass(frozen=True, slots=True)
+class Received:
+    """A message arrived."""
+
+    data: object
+
+
+@dataclass(frozen=True, slots=True)
+class Unreadable:
+    """A message arrived that could not be read (Node's ``messageerror``)."""
+
+    cause: BaseException
+
+
+@dataclass(frozen=True, slots=True)
+class Raised:
+    """The worker raised (Node's ``error`` event)."""
+
+    cause: BaseException
+
+
+@dataclass(frozen=True, slots=True)
+class Exited:
+    """The worker has exited."""
+
+    code: int
+
+
+@dataclass(frozen=True, slots=True)
+class Disconnected:
+    """The parent went away."""
+
+
+type WorkerEvent = Received | Unreadable | Raised | Exited
+type ParentEvent = Received | Unreadable | Disconnected
+
+
+class WorkerPort(ABC):
+    """The parent's end of one worker."""
+
+    __slots__ = ("_finished", "events", "exited")
+
+    def __init__(self) -> None:
+        self.events: Mailbox[WorkerEvent] = Mailbox()
+        self.exited: Latch[int] = Latch(waits_for="event")
+        self._finished = threading.Lock()
+
+    @abstractmethod
+    def post(self, message: object) -> None:
+        """Send ``message`` without waiting; raises if it cannot be sent."""
+
+    @abstractmethod
+    def kill(self) -> None:
+        """Stop the worker now."""
+
+    def finish(self, code: int) -> None:
+        """Report the exit, once."""
+        if self._finished.acquire(blocking=False):
+            self.events.put(Exited(code))
+            self.exited.set(code)
+
+
+class ParentPort(ABC):
+    """The worker's end of its parent."""
+
+    __slots__ = ("inbox",)
+
+    def __init__(self) -> None:
+        self.inbox: Mailbox[ParentEvent] = Mailbox()
+
+    @abstractmethod
+    def post(self, message: object) -> None:
+        """Send ``message`` to the parent without waiting."""
+
+    @abstractmethod
+    def close(self) -> None:
+        """Stop receiving: the parent closed the worker."""
+
+
+class WorkerHandle(ABC):
+    """A worker that has not started yet: what a ``Spawner`` gives the
+    Python platform. ``start`` runs it."""
+
+    __slots__ = ()
+
+    @abstractmethod
+    def start(self) -> WorkerPort: ...