pyfect-dev 0.1.0__py3-none-any.whl

pyfect/process/_child.py

@@ -0,0 +1,213 @@
+"""A spawned child process: its exit, its signals, and how it is killed
+(the process-group handling of Effect's ``NodeChildProcessSpawner``).
+
+A detached child (the default on POSIX) leads a process group of its own,
+and killing it signals the whole group. Without ``force_kill_after``, a kill
+waits for the leader to exit with no time limit, then gives the rest of the
+group up to a second; with it, ``SIGKILL`` follows after that long and the
+group gets a final second. Waits poll the group on the scheduler's
+real-time timers, never on the ``Clock``. A group whose leader has exited
+is found by its numeric id, so if that id were reused before cleanup an
+unrelated group could be signalled.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import errno
+import os
+import signal
+import subprocess
+import threading
+import time
+from collections.abc import Callable
+from typing import Never
+
+from pyfect._core import Effect
+from pyfect.duration import Seconds
+from pyfect.effect import callback, fail, gen, succeed, suspend, sync, try_
+from pyfect.platform_error import PlatformError, from_exception, system_error
+from pyfect.process._command import StandardCommand
+from pyfect.process._os import Latch
+from pyfect.scheduler import Cancel, CurrentScheduler, Scheduler
+
+GROUP_GRACE = Seconds(1.0)
+"""How long the rest of a process group gets to exit after its leader."""
+
+_POLL = Seconds(0.01)
+
+_SIGKILL: signal.Signals = getattr(signal, "SIGKILL", signal.SIGTERM)
+
+
+class Child:
+    """A started process, watched by a thread that waits for its exit."""
+
+    __slots__ = ("_exit", "command", "command_line", "pid", "popen", "referenced")
+
+    def __init__(
+        self, command: StandardCommand, popen: subprocess.Popen[bytes], command_line: str
+    ) -> None:
+        self.command = command
+        self.popen = popen
+        self.pid = popen.pid
+        self.command_line = command_line
+        self.referenced = True
+        self._exit: Latch[int] = Latch(waits_for="event")
+        threading.Thread(target=self._watch, name="pyfect-process-exit", daemon=True).start()
+
+    # --- state ------------------------------------------------------------------
+
+    def returncode(self) -> int | None:
+        """``subprocess``'s code once the leader has exited (``-N`` for
+        signal ``N``)."""
+        code = self._exit.get()
+        return None if code is None else code[0]
+
+    def exited(self) -> bool:
+        return self._exit.get() is not None
+
+    def after_exit(self, callback: Callable[[], None]) -> None:
+        """Call ``callback`` once the leader has exited (now if it has)."""
+        self._exit.on_set(callback)
+
+    def wait_exit(self) -> Effect[int]:
+        return self._exit.wait()
+
+    def owns_group(self) -> bool:
+        """A non-detached leader shares its parent's group, so once it has
+        exited there is no group of its own left."""
+        return os.name != "nt" and (self.command.options.detached is not False or not self.exited())
+
+    def alive(self) -> bool:
+        """The leader has not exited, or a member of its group still
+        exists."""
+        if not self.exited():
+            return True
+        if not self.owns_group():
+            return False
+        try:
+            os.killpg(self.pid, 0)
+        except PermissionError:
+            # macOS answers EPERM while the group's remaining members are
+            # zombies.
+            return True
+        except OSError:
+            return False
+        return True
+
+    def _watch(self) -> None:
+        code = self.popen.wait()
+        if code > 0 and self.command.options.detached is not False and os.name != "nt":
+            # Best effort, as Node's exit listener: a failed leader takes its
+            # group with it.
+            with contextlib.suppress(OSError):
+                os.killpg(self.pid, self.command.options.kill_signal or signal.SIGTERM)
+        self._exit.set(code)
+
+    # --- signals ----------------------------------------------------------------
+
+    def _signal_process(self, sig: signal.Signals) -> None:
+        if self.exited():
+            raise ProcessLookupError(errno.ESRCH, "Failed to kill child process")
+        os.kill(self.pid, sig)
+
+    def _signal_group(self, sig: signal.Signals) -> None:
+        if self.owns_group():
+            try:
+                os.killpg(self.pid, sig)
+                return
+            except OSError as error:
+                # Only zombies remain in an exited leader's group.
+                if self.exited() and error.errno == errno.EPERM:
+                    return
+        self._signal_process(sig)
+
+    def signal(self, sig: signal.Signals) -> Effect[None, PlatformError]:
+        return try_(lambda: self._signal_group(sig), self._error("kill"))
+
+    def _error(self, method: str) -> Callable[[Exception], PlatformError]:
+        return lambda error: from_exception(
+            error, module="ChildProcess", method=method, path_or_descriptor=self.command_line
+        )
+
+    def _await_group(self, timeout: Seconds) -> Effect[None]:
+        """Wait, at most ``timeout``, for the leader and its group to be
+        gone, polling on the scheduler's real-time timers."""
+
+        def start(scheduler: Scheduler) -> Effect[None]:
+            def register(resume: Callable[[Effect[None]], None]) -> Effect[None]:
+                deadline = time.monotonic() + timeout
+                pending: list[Cancel] = []
+
+                def poll() -> None:
+                    if time.monotonic() >= deadline or not self.alive():
+                        resume(succeed(None))
+                    else:
+                        pending[:] = [scheduler.call_later(_POLL, poll)]
+
+                def stop() -> None:
+                    for cancel in pending:
+                        cancel()
+
+                poll()
+                return sync(stop)
+
+            return callback(register)
+
+        return CurrentScheduler().flat_map(start)
+
+    def terminate(
+        self, kill_signal: signal.Signals | None, force_kill_after: Seconds | None
+    ) -> Effect[None, PlatformError]:
+        def body():
+            yield from self.signal(kill_signal or signal.SIGTERM)
+            if force_kill_after is None:
+                yield from self.wait_exit()
+                yield from self._await_group(GROUP_GRACE)
+            else:
+                yield from self._await_group(force_kill_after)
+                if self.alive():
+                    yield from self.signal(_SIGKILL)
+                    yield from self._await_group(GROUP_GRACE)
+            yield from self.wait_exit()
+
+        return gen(body)
+
+    def release(self) -> Effect[None]:
+        """What the scope does when it closes (Effect's release of the
+        spawn): kill a referenced process, and the group of an exited one
+        unless a successful non-detached leader left none."""
+        options = self.command.options
+
+        def kill() -> Effect[None]:
+            return self.terminate(options.kill_signal, options.force_kill_after).ignore()
+
+        def decide() -> Effect[None]:
+            code = self.returncode()
+            if code is None:
+                return kill() if self.referenced else succeed(None)
+            if code > 0:
+                return kill()
+            if self.referenced and os.name != "nt" and options.detached is not False:
+                return kill()
+            return succeed(None)
+
+        return suspend(decide)
+
+    def exit_code_error(self, code: int) -> Effect[Never, PlatformError]:
+        """A signal ended the leader: ``exit_code`` fails."""
+        name = signal.Signals(-code).name
+        return fail(
+            system_error(
+                "Unknown",
+                module="ChildProcess",
+                method="exit_code",
+                description=f"Process interrupted due to receipt of signal: '{name}'",
+                path_or_descriptor=self.command_line,
+            )
+        )
+
+
+def command_line(commands: tuple[StandardCommand, ...]) -> str:
+    """How errors name a pipeline: ``cmd args | cmd args``."""
+    return " | ".join(" ".join((c.command, *c.args)) for c in commands)

pyfect/process/_command.py

@@ -0,0 +1,358 @@
+"""Commands: descriptions of child processes before they start (Effect's
+``ChildProcess``).
+
+A command is an ``Effect``: running it asks the ``ChildProcessSpawner``
+service to start it, in the current ``Scope``, and gives back a
+``ChildProcessHandle``.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import signal
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field, replace
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, Literal, Never, final
+
+from pyfect._core import Effect
+from pyfect.duration import Duration, Seconds, to_seconds
+from pyfect.platform_error import PlatformError
+from pyfect.sink import Sink
+from pyfect.sink._sink import require_sink
+from pyfect.stream import Stream
+from pyfect.stream._base import require_stream
+
+if TYPE_CHECKING:
+    from pyfect._effect_base import Step
+    from pyfect._fiber import Fiber
+
+    # Named in the base class's quoted type arguments.
+    from pyfect.process._handle import ChildProcessHandle  # noqa: F401
+    from pyfect.process._spawner import ChildProcessSpawner  # noqa: F401
+    from pyfect.scope import Scope  # noqa: F401
+
+type StdioKind = Literal["pipe", "inherit", "ignore"]
+"""How the operating system sets up a standard stream: a pipe to the
+parent, the parent's own stream, or nothing (``/dev/null``)."""
+
+type CommandInput = StdioKind | Stream[bytes, PlatformError]
+"""A child's standard input: a ``StdioKind``, or a stream written into
+it."""
+
+type CommandOutput = StdioKind | Sink[bytes, bytes, Never, PlatformError]
+"""A child's standard output or error: a ``StdioKind``, or a sink its
+bytes are transduced through as the handle's stream is read."""
+
+type PipeFrom = Literal["stdout", "stderr", "all"] | int
+"""Which output of a command a pipe reads: an ``int`` is an additional
+output descriptor (3 or more)."""
+
+type PipeTo = Literal["stdin"] | int
+"""Which input of a command a pipe writes: an ``int`` is an additional
+input descriptor (3 or more)."""
+
+
+@dataclass(frozen=True, slots=True)
+class StdinConfig:
+    """A child's standard input; with ``end_on_done`` (the default) the pipe
+    closes once a given stream ends."""
+
+    stream: CommandInput = "pipe"
+    end_on_done: bool = True
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.stream, str):
+            require_stream(self.stream, "stream")
+
+
+@dataclass(frozen=True, slots=True)
+class InputFd:
+    """An additional descriptor the parent writes and the child reads,
+    fed by ``stream`` if given."""
+
+    stream: Stream[bytes, PlatformError] | None = None
+
+    def __post_init__(self) -> None:
+        if self.stream is not None:
+            require_stream(self.stream, "stream")
+
+
+@dataclass(frozen=True, slots=True)
+class OutputFd:
+    """An additional descriptor the child writes and the parent reads,
+    transduced through ``sink`` if given."""
+
+    sink: Sink[bytes, bytes, Never, PlatformError] | None = None
+
+    def __post_init__(self) -> None:
+        if self.sink is not None:
+            require_sink(self.sink, "sink")
+
+
+type AdditionalFd = InputFd | OutputFd
+
+_FIRST_EXTRA_FD = 3
+
+
+def _no_fds() -> Mapping[int, AdditionalFd]:
+    return MappingProxyType({})
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CommandOptions:
+    """How a command runs; see ``make`` for each option."""
+
+    cwd: str | None = None
+    env: Mapping[str, str | None] | None = None
+    extend_env: bool = False
+    shell: bool | str = False
+    detached: bool | None = None
+    windows_hide: bool | None = None
+    stdin: StdinConfig = StdinConfig()
+    stdout: CommandOutput = "pipe"
+    stderr: CommandOutput = "pipe"
+    additional_fds: Mapping[int, AdditionalFd] = field(default_factory=_no_fds)
+    kill_signal: signal.Signals | None = None
+    force_kill_after: Seconds | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class PipeOptions:
+    """Which output of the left command feeds which input of the right."""
+
+    from_: PipeFrom = "stdout"
+    to: PipeTo = "stdin"
+
+
+class _CommandBase(Effect["ChildProcessHandle", PlatformError, "ChildProcessSpawner | Scope"]):
+    __slots__ = ()
+
+    def _step(self, fiber: Fiber[Any, Any]) -> Step:
+        from pyfect.process._spawner import ChildProcessSpawner
+
+        command: Command = self  # pyright: ignore[reportAssignmentType]
+        return ChildProcessSpawner().flat_map(lambda spawner: spawner.spawn(command))
+
+    def pipe_to(
+        self, that: Command, *, from_: PipeFrom = "stdout", to: PipeTo = "stdin"
+    ) -> PipedCommand:
+        """Feed ``from_`` of this command (standard output by default) into
+        ``to`` of ``that`` (standard input by default)."""
+        if not isinstance(that, (StandardCommand, PipedCommand)):  # pyright: ignore[reportUnnecessaryIsInstance]
+            raise TypeError(f"that must be a command, got {type(that).__name__}")
+        if isinstance(from_, int) and from_ < _FIRST_EXTRA_FD:
+            from_ = "stdout"
+        if isinstance(to, int) and to < _FIRST_EXTRA_FD:
+            to = "stdin"
+        left: Command = self  # pyright: ignore[reportAssignmentType]
+        return PipedCommand(left, that, PipeOptions(from_=from_, to=to))
+
+    def prefix(self, command: str, args: Sequence[str] = ()) -> Command:
+        """Run this command through ``command args...`` (``time``, ``nice``,
+        ...); in a pipeline, the leftmost command only."""
+        this: Command = self  # pyright: ignore[reportAssignmentType]
+        return _prefix(this, command, tuple(args))
+
+    def set_cwd(self, cwd: str | os.PathLike[str]) -> Command:
+        """Run in ``cwd``; in a pipeline, every command."""
+        this: Command = self  # pyright: ignore[reportAssignmentType]
+        return _set_cwd(this, os.fspath(cwd))
+
+    def set_env(self, env: Mapping[str, str]) -> Command:
+        """Add ``env`` to the command's environment, replacing variables of
+        the same name; in a pipeline, every command's."""
+        this: Command = self  # pyright: ignore[reportAssignmentType]
+        return _set_env(this, dict(env))
+
+
+@final
+class StandardCommand(_CommandBase):
+    """One executable with its arguments and options."""
+
+    __slots__ = ("args", "command", "options")
+
+    def __init__(self, command: str, args: tuple[str, ...], options: CommandOptions) -> None:
+        self.command = command
+        self.args = args
+        self.options = options
+
+    def __repr__(self) -> str:
+        return f"StandardCommand({self.command!r}, {list(self.args)!r})"
+
+
+@final
+class PipedCommand(_CommandBase):
+    """``left``'s output feeding ``right``'s input."""
+
+    __slots__ = ("left", "options", "right")
+
+    def __init__(self, left: Command, right: Command, options: PipeOptions) -> None:
+        self.left = left
+        self.right = right
+        self.options = options
+
+    def __repr__(self) -> str:
+        return f"PipedCommand({self.left!r}, {self.right!r})"
+
+
+type Command = StandardCommand | PipedCommand
+
+
+def make(
+    command: str,
+    args: Sequence[str] = (),
+    *,
+    cwd: str | os.PathLike[str] | None = None,
+    env: Mapping[str, str | None] | None = None,
+    extend_env: bool = False,
+    shell: bool | str = False,
+    detached: bool | None = None,
+    windows_hide: bool | None = None,
+    stdin: CommandInput | StdinConfig = "pipe",
+    stdout: CommandOutput = "pipe",
+    stderr: CommandOutput = "pipe",
+    additional_fds: Mapping[int, AdditionalFd] | None = None,
+    kill_signal: signal.Signals | None = None,
+    force_kill_after: Duration | None = None,
+) -> StandardCommand:
+    """A command running ``command`` with ``args`` (Effect's array form of
+    ``ChildProcess.make``; ``shlex.split`` turns a command line into one).
+
+    - ``cwd``: the working directory, checked to exist when spawned.
+    - ``env``: the environment; it replaces the parent's unless
+      ``extend_env``, which adds it over the parent's. A ``None`` value
+      removes the variable.
+    - ``shell``: run ``command`` as a line of shell with ``/bin/sh``
+      (``cmd.exe`` on Windows) or the shell given; each of ``args`` is
+      quoted and appended as one literal word.
+    - ``detached``: start a process group of its own, so killing the
+      command kills its descendants (the default except on Windows).
+    - ``windows_hide``: hide the console window on Windows (the default
+      unless detached).
+    - ``stdin``, ``stdout``, ``stderr``: see ``CommandInput``,
+      ``CommandOutput`` and ``StdinConfig``; each is a pipe by default.
+    - ``additional_fds``: descriptors from 3 up, by number; lower ones are skipped.
+    - ``kill_signal``, ``force_kill_after``: how the scope ending the
+      command kills it: the signal (``SIGTERM`` by default), then
+      ``SIGKILL`` after this long if the process group is still alive.
+    """
+    stdin_config = stdin if isinstance(stdin, StdinConfig) else StdinConfig(_input(stdin, "stdin"))
+    fds = MappingProxyType(
+        {fd: config for fd, config in (additional_fds or {}).items() if fd >= _FIRST_EXTRA_FD}
+    )
+    for fd, config in fds.items():
+        if not isinstance(config, (InputFd, OutputFd)):  # pyright: ignore[reportUnnecessaryIsInstance]
+            raise TypeError(f"additional_fds[{fd}] must be an InputFd or OutputFd")
+    options = CommandOptions(
+        cwd=None if cwd is None else os.fspath(cwd),
+        env=None if env is None else dict(env),
+        extend_env=extend_env,
+        shell=shell,
+        detached=detached,
+        windows_hide=windows_hide,
+        stdin=stdin_config,
+        stdout=_output(stdout, "stdout"),
+        stderr=_output(stderr, "stderr"),
+        additional_fds=fds,
+        kill_signal=kill_signal,
+        force_kill_after=None if force_kill_after is None else to_seconds(force_kill_after),
+    )
+    return StandardCommand(command, tuple(args), options)
+
+
+def with_options(command: StandardCommand, **changes: Any) -> StandardCommand:
+    """``command`` with some of its ``CommandOptions`` replaced."""
+    return StandardCommand(command.command, command.args, replace(command.options, **changes))
+
+
+def _input(value: CommandInput, name: str) -> CommandInput:
+    if not isinstance(value, str):
+        require_stream(value, name)
+    return value
+
+
+def _output(value: CommandOutput, name: str) -> CommandOutput:
+    if not isinstance(value, str):
+        require_sink(value, name)
+    return value
+
+
+def _prefix(command: Command, executable: str, args: tuple[str, ...]) -> Command:
+    match command:
+        case StandardCommand():
+            return StandardCommand(
+                executable, (*args, command.command, *command.args), command.options
+            )
+        case PipedCommand():
+            return PipedCommand(
+                _prefix(command.left, executable, args), command.right, command.options
+            )
+
+
+def _set_cwd(command: Command, cwd: str) -> Command:
+    match command:
+        case StandardCommand():
+            return with_options(command, cwd=cwd)
+        case PipedCommand():
+            return PipedCommand(
+                _set_cwd(command.left, cwd), _set_cwd(command.right, cwd), command.options
+            )
+
+
+def _set_env(command: Command, env: dict[str, str]) -> Command:
+    match command:
+        case StandardCommand():
+            merged = env if command.options.env is None else {**command.options.env, **env}
+            return with_options(command, env=merged)
+        case PipedCommand():
+            return PipedCommand(
+                _set_env(command.left, env), _set_env(command.right, env), command.options
+            )
+
+
+@dataclass(frozen=True, slots=True)
+class FlattenedPipeline:
+    """A pipeline's commands from left to right, and the options of each
+    pipe between two of them."""
+
+    commands: tuple[StandardCommand, *tuple[StandardCommand, ...]]
+    pipe_options: tuple[PipeOptions, ...]
+
+
+def flatten_command(command: Command) -> FlattenedPipeline:
+    """The commands of a pipeline in order, with the options of each pipe."""
+    commands: list[StandardCommand] = []
+    pipes: list[PipeOptions] = []
+
+    def walk(cmd: Command) -> None:
+        match cmd:
+            case StandardCommand():
+                commands.append(cmd)
+            case PipedCommand():
+                walk(cmd.left)
+                pipes.append(cmd.options)
+                walk(cmd.right)
+
+    walk(command)
+    first, *rest = commands
+    return FlattenedPipeline((first, *rest), tuple(pipes))
+
+
+_FD_NAME = re.compile(r"fd([0-9]+)")
+
+
+def parse_fd_name(name: str) -> int | None:
+    """The descriptor ``"fd3"`` names (3), or ``None`` unless the name is
+    ``fd`` and a number from 3 up."""
+    match = _FD_NAME.fullmatch(name)
+    if match is None:
+        return None
+    fd = int(match[1])
+    return fd if fd >= 3 else None
+
+
+def fd_name(fd: int) -> str:
+    """``"fd3"`` for 3."""
+    return f"fd{fd}"

pyfect/process/_handle.py

@@ -0,0 +1,77 @@
+"""A running child process (Effect's ``ChildProcessHandle``)."""
+
+from __future__ import annotations
+
+import signal
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Never, NewType, Protocol
+
+from pyfect._core import Effect
+from pyfect._effect_args import require_effect
+from pyfect.duration import Duration
+from pyfect.platform_error import PlatformError
+from pyfect.sink import Sink
+from pyfect.sink._sink import require_sink
+from pyfect.stream import Stream
+from pyfect.stream._base import require_stream
+
+ExitCode = NewType("ExitCode", int)
+"""The status a child process exited with."""
+
+ProcessId = NewType("ProcessId", int)
+"""A child process's operating-system identifier."""
+
+type Reref = Effect[None, PlatformError]
+"""Given back by ``ChildProcessHandle.unref``: running it makes the scope
+own the process again."""
+
+
+class Kill(Protocol):
+    """Kills a child process with ``kill_signal`` (``SIGTERM`` by default)
+    and waits for it. With ``force_kill_after``, sends ``SIGKILL`` if the
+    process (group) is still alive after that long; without it, waits with
+    no time limit."""
+
+    def __call__(
+        self, *, kill_signal: signal.Signals | None = None, force_kill_after: Duration | None = None
+    ) -> Effect[None, PlatformError]: ...
+
+
+@dataclass(frozen=True, slots=True, kw_only=True, eq=False)
+class ChildProcessHandle:
+    """A running child process; the constructor is Effect's ``makeHandle``.
+
+    - ``exit_code``: waits for the exit; fails if a signal ended the
+      process, or with the error of a stream feeding one of its inputs.
+    - ``stdin``: a sink writing standard input.
+    - ``stdout``, ``stderr``: byte streams of its output; ``all`` both,
+      interleaved. Reading one alongside ``all`` splits the output between
+      them.
+    - ``get_input_fd`` / ``get_output_fd``: an additional descriptor's sink
+      or stream (a draining sink and an empty stream for one not set up).
+    - ``unref``: stop the scope from killing the process when it closes;
+      the ``Reref`` it gives back undoes that.
+    """
+
+    pid: ProcessId
+    exit_code: Effect[ExitCode, PlatformError]
+    is_running: Effect[bool, PlatformError]
+    kill: Kill
+    stdin: Sink[None, bytes, Never, PlatformError]
+    stdout: Stream[bytes, PlatformError]
+    stderr: Stream[bytes, PlatformError]
+    all: Stream[bytes, PlatformError]
+    get_input_fd: Callable[[int], Sink[None, bytes, Never, PlatformError]]
+    get_output_fd: Callable[[int], Stream[bytes, PlatformError]]
+    unref: Effect[Reref, PlatformError]
+
+    def __post_init__(self) -> None:
+        for name in ("exit_code", "is_running", "unref"):
+            require_effect(getattr(self, name), name)
+        require_sink(self.stdin, "stdin")
+        for name in ("stdout", "stderr", "all"):
+            require_stream(getattr(self, name), name)
+
+    def __repr__(self) -> str:
+        return f"ChildProcessHandle(pid={self.pid})"