pyfect-dev 0.1.0__py3-none-any.whl

pyfect/process/_pipes.py

@@ -0,0 +1,291 @@
+"""The parent's ends of a child's pipes, as byte streams and sinks.
+
+A ``Reader`` has a thread that reads only while a fiber waits for data, so
+an unread pipe fills up and the child waits for it, as a Node stream's
+backpressure makes it. Chunks it has read wait in the reader until taken,
+so an interrupted pull loses nothing. A ``Writer`` has a thread, started
+by its first write, that writes in short waits on a non-blocking
+descriptor, so an interrupted write stops within one of those waits. Each
+pipe has threads of its own, never a shared pool, so a pipe waiting on its
+child never holds up another pipe. Both close their descriptor themselves,
+once no thread is using it.
+"""
+
+from __future__ import annotations
+
+import errno
+import os
+import select
+import threading
+from collections import deque
+from collections.abc import Callable
+from typing import Never
+
+from pyfect import sink as sink_
+from pyfect import stream as stream_
+from pyfect._core import Effect
+from pyfect._os_wait import Cancel, Resume, os_wait
+from pyfect.effect import fail, succeed, suspend, sync
+from pyfect.errors import Done
+from pyfect.platform_error import PlatformError
+from pyfect.sink import Sink
+from pyfect.stream import Stream
+
+CHUNK_SIZE = 64 * 1024
+_POLL = 0.05
+"""Seconds a pipe thread waits for its descriptor before checking whether
+it was closed."""
+
+type OnError = Callable[[OSError], PlatformError]
+type _Pull = Effect[tuple[bytes, ...], PlatformError | Done]
+
+
+def _ready(fd: int, *, write: bool) -> bool:
+    if os.name == "nt":
+        return True
+    ready = select.select([], [fd], [], _POLL) if write else select.select([fd], [], [], _POLL)
+    return bool(ready[0] or ready[1])
+
+
+class Reader:
+    """Reads the pipe ``fd`` on demand; ``stream`` is its bytes."""
+
+    __slots__ = (
+        "_chunks",
+        "_closing",
+        "_cond",
+        "_ended",
+        "_error",
+        "_fd",
+        "_on_error",
+        "_waiters",
+    )
+
+    def __init__(self, fd: int, on_error: OnError) -> None:
+        self._fd = fd
+        self._on_error = on_error
+        self._cond = threading.Condition()
+        self._chunks: deque[bytes] = deque()
+        self._ended = False
+        self._closing = False
+        self._error: OSError | None = None
+        self._waiters: list[Resume[tuple[bytes, ...], PlatformError | Done]] = []
+        threading.Thread(target=self._run, name="pyfect-process-reader", daemon=True).start()
+
+    @property
+    def stream(self) -> Stream[bytes, PlatformError]:
+        return stream_.from_pull(succeed(self._pull()))
+
+    def close(self) -> None:
+        """Stop reading once what the pipe holds now is read (all a child
+        that has exited wrote); then pulls end the stream."""
+        with self._cond:
+            self._closing = True
+            self._cond.notify()
+
+    def _pull(self) -> _Pull:
+        def register(resume: Resume[tuple[bytes, ...], PlatformError | Done]) -> Cancel | None:
+            with self._cond:
+                taken = self._take()
+                if taken is None:
+                    self._waiters.append(resume)
+                    self._cond.notify()
+            if taken is not None:
+                resume(taken)
+                return None
+            return lambda: self._forget(resume)
+
+        return os_wait(register, waits_for="event")
+
+    def _take(self) -> _Pull | None:
+        if self._chunks:
+            return succeed((self._chunks.popleft(),))
+        if self._error is not None:
+            return fail(self._on_error(self._error))
+        if self._ended:
+            return fail(Done())
+        return None
+
+    def _forget(self, resume: Resume[tuple[bytes, ...], PlatformError | Done]) -> None:
+        with self._cond:
+            if resume in self._waiters:
+                self._waiters.remove(resume)
+
+    def _wake(self, waiters: list[Resume[tuple[bytes, ...], PlatformError | Done]]) -> None:
+        # Woken pulls run again rather than being handed a chunk: one that
+        # was interrupted meanwhile takes nothing.
+        for resume in waiters:
+            resume(suspend(self._pull))
+
+    def _wanted(self) -> bool:
+        return bool(self._waiters) and not self._chunks
+
+    def _run(self) -> None:
+        try:
+            while self._read_once():
+                pass
+        finally:
+            os.close(self._fd)
+
+    def _read_once(self) -> bool:
+        with self._cond:
+            while not self._ended and not self._closing and not self._wanted():
+                self._cond.wait()
+            if self._ended:
+                return False
+            closing = self._closing
+        if closing:
+            self._end(*self._drain())
+            return False
+        data = b""
+        error: OSError | None = None
+        try:
+            if not _ready(self._fd, write=False):
+                return True
+            data = os.read(self._fd, CHUNK_SIZE)
+        except OSError as e:
+            error = e
+        if not data:
+            self._end([], error)
+            return False
+        with self._cond:
+            self._chunks.append(data)
+            waiters, self._waiters = self._waiters, []
+        self._wake(waiters)
+        return True
+
+    def _drain(self) -> tuple[list[bytes], OSError | None]:
+        """What the pipe holds now, without waiting for more."""
+        chunks: list[bytes] = []
+        if os.name == "nt":
+            return chunks, None
+        try:
+            while select.select([self._fd], [], [], 0)[0]:
+                data = os.read(self._fd, CHUNK_SIZE)
+                if not data:
+                    break
+                chunks.append(data)
+        except OSError as error:
+            return chunks, error
+        return chunks, None
+
+    def _end(self, chunks: list[bytes], error: OSError | None) -> None:
+        with self._cond:
+            self._chunks.extend(chunks)
+            self._error = error
+            self._ended = True
+            waiters, self._waiters = self._waiters, []
+        self._wake(waiters)
+
+
+class _Write:
+    """One write waiting for, or held by, a writer's thread."""
+
+    __slots__ = ("cancelled", "resume", "view")
+
+    def __init__(self, data: bytes, resume: Resume[None, PlatformError]) -> None:
+        self.view = memoryview(data)
+        self.resume = resume
+        self.cancelled = False
+
+    def cancel(self) -> None:
+        """Skip the rest of the write: its thread stops at its next wait or
+        never starts it."""
+        self.cancelled = True
+
+
+class Writer:
+    """Writes to the pipe ``fd``; ``sink`` writes its input."""
+
+    __slots__ = ("_closed", "_cond", "_fd", "_on_error", "_queue", "_started")
+
+    def __init__(self, fd: int, on_error: OnError) -> None:
+        self._fd = fd
+        self._on_error = on_error
+        self._cond = threading.Condition()
+        self._queue: deque[_Write] = deque()
+        self._closed = False
+        self._started = False
+        if os.name != "nt":
+            os.set_blocking(fd, False)
+
+    def sink(self, *, end_on_done: bool) -> Sink[None, bytes, Never, PlatformError]:
+        """Write each chunk; with ``end_on_done``, close the pipe however the
+        input ends."""
+        write: Sink[None, bytes, Never, PlatformError] = sink_.for_each(self.write)
+        return write.ensuring(sync(self.close)) if end_on_done else write
+
+    def write(self, data: bytes) -> Effect[None, PlatformError]:
+        def register(resume: Resume[None, PlatformError]) -> Cancel | None:
+            request = _Write(data, resume)
+            with self._cond:
+                closed = self._closed
+                if not closed:
+                    self._queue.append(request)
+                    self._start()
+                    self._cond.notify()
+            if closed:
+                resume(fail(self._on_error(BrokenPipeError(errno.EPIPE, "write after end"))))
+                return None
+            return request.cancel
+
+        return os_wait(register, waits_for="work")
+
+    def close(self) -> None:
+        """End the pipe: a write in progress stops, queued ones fail."""
+        with self._cond:
+            if self._closed:
+                return
+            self._closed = True
+            started = self._started
+            self._cond.notify()
+        if not started:
+            os.close(self._fd)
+
+    def _start(self) -> None:
+        if not self._started:
+            self._started = True
+            threading.Thread(target=self._run, name="pyfect-process-writer", daemon=True).start()
+
+    def _cancel(self, request: _Write) -> None:
+        with self._cond:
+            request.cancelled = True
+            # The request in progress stops itself; a queued one leaves.
+            if request in self._queue and request is not self._queue[0]:
+                self._queue.remove(request)
+
+    def _run(self) -> None:
+        try:
+            while (request := self._next()) is not None:
+                self._serve(request)
+        finally:
+            os.close(self._fd)
+
+    def _next(self) -> _Write | None:
+        with self._cond:
+            while not self._queue and not self._closed:
+                self._cond.wait()
+            if not self._closed:
+                return self._queue[0]
+            queued, self._queue = list(self._queue), deque()
+        for request in queued:
+            request.resume(fail(self._on_error(BrokenPipeError(errno.EPIPE, "write after end"))))
+        return None
+
+    def _serve(self, request: _Write) -> None:
+        """Write ``request`` in short waits, until done, cancelled or
+        closed."""
+        result: Effect[None, PlatformError] = succeed(None)
+        try:
+            while request.view and not request.cancelled and not self._closed:
+                if not _ready(self._fd, write=True):
+                    continue
+                try:
+                    request.view = request.view[os.write(self._fd, request.view) :]
+                except BlockingIOError:
+                    continue
+        except OSError as error:
+            result = fail(self._on_error(error))
+        with self._cond:
+            self._queue.popleft()
+        request.resume(result)

pyfect/process/_spawner.py

@@ -0,0 +1,75 @@
+"""The service that starts child processes (Effect's
+``ChildProcessSpawner``)."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+
+from pyfect._core import Effect
+from pyfect.context import Tag
+from pyfect.effect import never
+from pyfect.platform_error import PlatformError
+from pyfect.process._command import Command
+from pyfect.process._handle import ChildProcessHandle, ExitCode
+from pyfect.scope import Scope, scoped
+from pyfect.stream import Stream
+from pyfect.stream import unwrap as unwrap_stream
+
+type Spawn = Callable[[Command], Effect[ChildProcessHandle, PlatformError, Scope]]
+
+
+@dataclass(frozen=True, slots=True)
+class ChildProcessSpawnerService:
+    """Starts commands; the constructor is Effect's
+    ``ChildProcessSpawner.make``: given ``spawn``, the rest is built on it.
+
+    The helpers own the handles they spawn, so they read and drop the output
+    they do not return, keeping a full pipe from stopping the child."""
+
+    spawn: Spawn
+
+    def exit_code(self, command: Command) -> Effect[ExitCode, PlatformError]:
+        """Run ``command`` and give back its exit code, reading and dropping
+        its output meanwhile. Returns once the process exits, even if
+        descendants keep its pipes open."""
+
+        def wait(handle: ChildProcessHandle) -> Effect[ExitCode, PlatformError]:
+            return handle.exit_code.race_first(handle.all.run_drain().flat_map(lambda _: never))
+
+        return scoped(self.spawn(command).flat_map(wait))
+
+    def stream_string(
+        self, command: Command, *, include_stderr: bool = False
+    ) -> Stream[str, PlatformError]:
+        """The standard output of ``command`` as text; with
+        ``include_stderr``, interleaved with its standard error, else that
+        is read and dropped until standard output ends."""
+
+        def text(handle: ChildProcessHandle) -> Stream[str, PlatformError]:
+            output = handle.all if include_stderr else handle.stdout.drain_fork(handle.stderr)
+            return output.decode_text()
+
+        return unwrap_stream(self.spawn(command).map(text))
+
+    def stream_lines(
+        self, command: Command, *, include_stderr: bool = False
+    ) -> Stream[str, PlatformError]:
+        """``stream_string`` split into lines."""
+        return self.stream_string(command, include_stderr=include_stderr).split_lines()
+
+    def lines(
+        self, command: Command, *, include_stderr: bool = False
+    ) -> Effect[list[str], PlatformError]:
+        """Run ``command`` and collect the lines of its output."""
+        return self.stream_lines(command, include_stderr=include_stderr).run_collect()
+
+    def string(
+        self, command: Command, *, include_stderr: bool = False
+    ) -> Effect[str, PlatformError]:
+        """Run ``command`` and give back its output."""
+        return self.stream_string(command, include_stderr=include_stderr).run_string()
+
+
+class ChildProcessSpawner(Tag[ChildProcessSpawnerService]):
+    """The service commands run with."""

pyfect/pubsub/__init__.py

@@ -0,0 +1,33 @@
+"""``PubSub[A]``: a hub that delivers every published message to every
+subscriber.
+
+A subscriber reads the messages published after it subscribed (plus up to
+``replay`` earlier ones), in order, through a ``Subscription`` that lasts
+until its scope closes. A message is stored once and dropped when every
+subscriber that owed it has read it. What happens when ``capacity``
+messages are unread is fixed at creation:
+
+- ``bounded``: publishers wait until the slowest subscriber reads.
+- ``dropping``: the new message is dropped and the publish returns ``False``.
+- ``sliding``: the oldest unread message is dropped for everyone.
+- ``unbounded``: the hub is never full.
+
+A message published while nobody is subscribed goes only to the replay
+buffer. ``end(value)`` stops publishing: subscribers read what is left, and
+then every ``take`` succeeds with ``value``. ``shutdown()`` unsubscribes
+everyone, interrupting their takes; later publishes return ``False``.
+
+Takes waiting on one subscription are served in the order they started.
+
+``Atomic`` is the store under a hub, with no waiting, for building a hub
+with ``PubSub.make``. The ``unsafe_*`` methods act at once, outside an
+effect; call them on the scheduler thread of the hub's fibers.
+"""
+
+from pyfect.pubsub._atomic import Atomic, BackingSubscription
+from pyfect.pubsub._pubsub import PubSub
+from pyfect.pubsub._replay import ReplayWindow
+from pyfect.pubsub._subscription import Subscription
+from pyfect.queue import Strategy
+
+__all__ = ["Atomic", "BackingSubscription", "PubSub", "ReplayWindow", "Strategy", "Subscription"]

pyfect/pubsub/_atomic.py

@@ -0,0 +1,181 @@
+"""``Atomic``: the message store under a ``PubSub``, with no waiting.
+
+Each published message is stored once with a count of the subscribers that
+still have to read it, and dropped when the count reaches 0. A message
+published while nobody is subscribed is not stored (only replayed). A
+subscriber reads messages published after it subscribed, in order.
+
+Counts reach 0 oldest first: every subscriber still owing a message also
+owes every later one. So the store is a deque whose head is the oldest
+message anyone still has to read, and ``slide`` drops that head for
+everyone.
+
+Effect's bounded variants (single slot, power of two, arbitrary ring) and
+its unbounded linked list behave alike; one deque with an optional capacity
+does for all of them.
+"""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+from collections import deque
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+from pyfect.option import Option, Some
+from pyfect.pubsub._replay import ReplayBuffer, ReplayWindow
+
+
+@dataclass(eq=False, slots=True)
+class _Slot[A]:
+    value: A
+    unread: int
+    replay_index: int
+
+
+class Atomic[A]:
+    """A store of messages for a set of subscribers. Not thread-safe: use it
+    from one scheduler thread, as a ``PubSub`` does."""
+
+    __slots__ = ("_capacity", "_replay", "_slots", "_start", "_subscribers")
+
+    def __init__(self, capacity: int | None, replay: int | None = None) -> None:
+        if capacity is not None and capacity <= 0:
+            raise ValueError(f"Cannot construct PubSub with capacity of {capacity}")
+        self._capacity = capacity
+        self._replay = ReplayBuffer[A](replay) if replay is not None and replay > 0 else None
+        self._slots: deque[_Slot[A]] = deque()
+        # The publish index of the oldest stored message.
+        self._start = 0
+        self._subscribers = 0
+
+    @classmethod
+    def bounded(cls, capacity: int | None, *, replay: int | None = None) -> Atomic[A]:
+        """A store of at most ``capacity`` messages (``None``: unbounded),
+        replaying the last ``replay`` to new subscribers (Effect's
+        ``makeAtomicBounded``). A capacity below 1 is a ``ValueError``."""
+        return cls(capacity, replay)
+
+    @classmethod
+    def unbounded(cls, *, replay: int | None = None) -> Atomic[A]:
+        """Effect's ``makeAtomicUnbounded``."""
+        return cls(None, replay)
+
+    @property
+    def capacity(self) -> int | None:
+        return self._capacity
+
+    def size(self) -> int:
+        """How many messages are stored (each once, however many readers)."""
+        return len(self._slots)
+
+    def is_empty(self) -> bool:
+        return not self._slots
+
+    def is_full(self) -> bool:
+        return len(self._slots) == self._capacity
+
+    def publish(self, value: A) -> bool:
+        """Store ``value`` for the current subscribers; ``False`` if full."""
+        if self.is_full():
+            return False
+        replay_index = 0 if self._replay is None else self._replay.offer(value)
+        if self._subscribers:
+            self._slots.append(_Slot(value, self._subscribers, replay_index))
+        return True
+
+    def publish_all(self, items: Iterable[A]) -> list[A]:
+        """Store what fits, in order; the items that did not."""
+        if not self._subscribers:
+            if self._replay is not None:
+                for item in items:
+                    self._replay.offer(item)
+            return []
+        batch = list(items)
+        fits = len(batch) if self._capacity is None else self._capacity - len(self._slots)
+        for item in batch[:fits]:
+            self.publish(item)
+        return batch[fits:]
+
+    def slide(self) -> None:
+        """Drop the oldest stored message for every subscriber."""
+        if self._slots:
+            slot = self._slots.popleft()
+            self._start += 1
+            if self._replay is not None:
+                self._replay.slide(slot.value, slot.replay_index)
+
+    def subscribe(self) -> BackingSubscription[A]:
+        """A reader of the messages published from now on."""
+        self._subscribers += 1
+        return BackingSubscription(self)
+
+    def replay_window(self) -> ReplayWindow[A]:
+        """The messages a subscriber that joins now replays."""
+        return ReplayWindow(self._replay)
+
+    # --- for BackingSubscription ------------------------------------------
+
+    def _end(self) -> int:
+        """The publish index the next stored message gets."""
+        return self._start + len(self._slots)
+
+    def _read(self, index: int) -> A:
+        """Read the message at publish ``index`` for one subscriber."""
+        slot = self._slots[index - self._start]
+        slot.unread -= 1
+        if slot.unread == 0:
+            self._slots.popleft()
+            self._start += 1
+        return slot.value
+
+    def _leave(self, index: int) -> None:
+        """A subscriber at ``index`` leaves without reading the rest."""
+        self._subscribers -= 1
+        for i in range(max(index, self._start), self._end()):
+            self._read(i)
+
+
+class BackingSubscription[A]:
+    """One subscriber's position in an ``Atomic``."""
+
+    __slots__ = ("_atomic", "_index", "_unsubscribed")
+
+    def __init__(self, atomic: Atomic[A]) -> None:
+        self._atomic = atomic
+        self._index = atomic._end()
+        self._unsubscribed = False
+
+    def size(self) -> int:
+        """How many messages are there to read."""
+        if self._unsubscribed:
+            return 0
+        atomic = self._atomic
+        return atomic._end() - max(self._index, atomic._start)
+
+    def is_empty(self) -> bool:
+        return self.size() == 0
+
+    def poll(self) -> Option[A]:
+        """The next message, if there is one."""
+        items = self.poll_up_to(1)
+        return Some(items[0]) if items else None
+
+    def poll_up_to(self, n: int) -> list[A]:
+        """Up to ``n`` messages (none if ``n`` is 0 or less)."""
+        count = min(n, self.size())
+        if count <= 0:
+            return []
+        atomic = self._atomic
+        start = max(self._index, atomic._start)
+        self._index = start + count
+        return [atomic._read(i) for i in range(start, start + count)]
+
+    def poll_all(self) -> list[A]:
+        return self.poll_up_to(self.size())
+
+    def unsubscribe(self) -> None:
+        """Stop reading; messages only this subscriber owed are dropped."""
+        if not self._unsubscribed:
+            self._unsubscribed = True
+            self._atomic._leave(self._index)