pyfect-dev 0.1.0__py3-none-any.whl

pyfect/_loop_thread.py

@@ -0,0 +1,152 @@
+"""An asyncio loop on a daemon thread of its own, owning the close of
+every async generator it runs.
+
+asyncio closes a generator dropped mid-iteration (httpx's response
+bodies, when a fiber lets go of a response) with a task it schedules from
+whichever thread dropped it, and keeps no reference to that task; a loop
+stopped and closed right after loses that close half done ("Task was
+destroyed but it is pending", "coroutine method 'aclose' ... was never
+awaited"). A ``LoopThread`` keeps each close task until it ends, and
+``stop`` closes the loop only once its tasks and generators have settled.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+import threading
+import weakref
+from collections.abc import AsyncGenerator, Callable, Coroutine
+from concurrent.futures import Future
+from typing import Any
+
+type ExceptionHandler = Callable[[asyncio.AbstractEventLoop, dict[str, Any]], object]
+type _AnyGenerator = AsyncGenerator[Any, Any]
+
+
+class LoopThread:
+    """An asyncio loop running on a daemon thread named ``name`` until
+    ``stop``. ``submit`` runs a coroutine on it; at ``stop``, a coroutine
+    still running is cancelled, and every async generator the loop ran is
+    closed before the loop closes (``stopped`` is then done). A generator
+    dropped after that is discarded without its close."""
+
+    __slots__ = (
+        "_closed",
+        "_closing",
+        "_generators",
+        "_lock",
+        "_stop",
+        "_tasks",
+        "loop",
+        "stopped",
+    )
+
+    def __init__(self, name: str, *, exception_handler: ExceptionHandler | None = None) -> None:
+        self.loop = asyncio.new_event_loop()
+        if exception_handler is not None:
+            self.loop.set_exception_handler(exception_handler)
+        self.stopped: Future[None] = Future()
+        """Done once the loop has closed."""
+        self._lock = threading.RLock()
+        """Reentrant: a generator may be dropped (and finalized) on the loop
+        thread while it holds the lock."""
+        self._closed = False
+        self._generators: weakref.WeakSet[_AnyGenerator] = weakref.WeakSet()
+        self._closing: set[_AnyGenerator] = set()
+        """Generators dropped mid-iteration, kept until their close ends."""
+        self._tasks: set[asyncio.Task[None]] = set()
+        self._stop = self.loop.create_future()
+        threading.Thread(target=self._run, name=name, daemon=True).start()
+
+    def submit[A](self, make: Callable[[], Coroutine[Any, Any, A]]) -> Future[A]:
+        """Run ``make()`` on the loop."""
+        return asyncio.run_coroutine_threadsafe(make(), self.loop)
+
+    def stop(self) -> None:
+        """Cancel what still runs, close the generators, then close the
+        loop (from any thread; stopping again does nothing)."""
+
+        def stop() -> None:
+            if not self._stop.done():
+                self._stop.set_result(None)
+
+        try:
+            self.loop.call_soon_threadsafe(stop)
+        except RuntimeError:
+            if not self.loop.is_closed():
+                raise
+
+    def _run(self) -> None:
+        # Running: a waiter giving up on ``stopped`` cannot cancel it.
+        self.stopped.set_running_or_notify_cancel()
+        asyncio.set_event_loop(self.loop)
+        try:
+            self.loop.run_until_complete(self._main())
+        finally:
+            self.loop.close()
+            self.stopped.set_result(None)
+
+    async def _main(self) -> None:
+        # Set inside the run: ``run_forever`` installs the loop's own hooks
+        # when it starts and restores the thread's previous ones at its end.
+        sys.set_asyncgen_hooks(firstiter=self._generators.add, finalizer=self._finalize)
+        await self._stop
+        await self._settle()
+        await self.loop.shutdown_default_executor()
+
+    def _finalize(self, generator: _AnyGenerator) -> None:
+        """A generator the loop ran was dropped before it ended (on any
+        thread): close it on the loop, unless the loop is closing."""
+        with self._lock:
+            if self._closed:
+                return
+            self._generators.discard(generator)
+            self._closing.add(generator)
+        self.loop.call_soon_threadsafe(self._start_close, generator)
+
+    def _start_close(self, generator: _AnyGenerator) -> None:
+        task = self.loop.create_task(self._close(generator))
+        self._tasks.add(task)
+        task.add_done_callback(self._tasks.discard)
+
+    async def _close(self, generator: _AnyGenerator) -> None:
+        try:
+            await generator.aclose()
+        except Exception as error:
+            self.loop.call_exception_handler(
+                {
+                    "message": "an error occurred during closing of asynchronous generator",
+                    "exception": error,
+                    "asyncgen": generator,
+                }
+            )
+        finally:
+            with self._lock:
+                self._closing.discard(generator)
+
+    async def _settle(self) -> None:
+        """Until nothing is left: cancel the coroutines still running (a
+        generator's close runs to its end), then close the generators still
+        open; each round may drop more generators."""
+        current = asyncio.current_task()
+        while True:
+            # Let closes scheduled from other threads start.
+            await asyncio.sleep(0)
+            generators: list[_AnyGenerator] = []
+            with self._lock:
+                running = [task for task in asyncio.all_tasks() if task is not current]
+                if not running and not self._closing:
+                    generators = list(self._generators)
+                    self._generators.clear()
+                    if not generators:
+                        self._closed = True
+                        return
+            for task in running:
+                if task not in self._tasks:
+                    task.cancel()
+            closes = [self._close(generator) for generator in generators]
+            await asyncio.gather(*running, *closes, return_exceptions=True)
+
+
+__all__ = ["ExceptionHandler", "LoopThread"]

pyfect/_metric_ref.py

@@ -0,0 +1,29 @@
+"""``FiberRuntimeMetrics``, apart from ``pyfect.metric`` so the fiber
+runtime can read it without importing the metrics."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Protocol
+
+from pyfect.context import Reference
+
+if TYPE_CHECKING:
+    from pyfect.context import Context
+    from pyfect.exit import Exit
+
+
+class FiberRuntimeMetricsService(Protocol):
+    """Told when each fiber starts and ends, with the fiber's context."""
+
+    def record_fiber_start(self, context: Context) -> None: ...
+
+    def record_fiber_end(self, context: Context, exit: Exit[Any, Any]) -> None: ...
+
+
+class FiberRuntimeMetrics(Reference[FiberRuntimeMetricsService | None]):
+    """What records fiber starts and ends; ``None`` (nothing) by default.
+    ``metric.enable_runtime_metrics`` provides ``metric.fiber_runtime_metrics``."""
+
+    @classmethod
+    def default(cls) -> FiberRuntimeMetricsService | None:
+        return None

pyfect/_one_or_many.py

@@ -0,0 +1,14 @@
+"""Constructors that take their items as arguments or, as Effect v4 does, as
+one array (``Schema.Literals([...])``, ``Schema.Union([...])``,
+``Config.Literals([...])``)."""
+
+from collections.abc import Sequence
+from typing import Any, cast
+
+
+def one_or_many(args: tuple[Any, ...]) -> tuple[Any, ...]:
+    """``args``, or the items of ``args``' only item when it is a ``list`` or
+    ``tuple``."""
+    if len(args) == 1 and isinstance(args[0], list | tuple):
+        return tuple(cast("Sequence[Any]", args[0]))
+    return args

pyfect/_os_wait.py

@@ -0,0 +1,118 @@
+"""Waiting for something outside the scheduler (a thread, another loop)
+without blocking a fiber.
+
+The fiber suspends in ``os_wait`` and the outside resumes it through the
+scheduler, from any thread. While it waits, a timer stays pending on the
+fiber's scheduler, so a synchronous run (a test's stall detection
+included) knows the wait is outside work, not a deadlock. Every wait on
+another thread goes through here (``tests/test_os_wait_guard.py``).
+
+Each wait says what it waits for, and the test clock treats the two apart:
+
+- ``"work"``: work the wait itself started, which finishes on its own (a
+  call on a thread, a query, a request). The scheduler counts it in its
+  ``outside_work``, and its resume too, so ``TestClock.adjust`` waits for it
+  before moving time: thread-backed work takes no virtual time, as Effect's
+  synchronous drivers do.
+- ``"event"``: something that may never come (a socket read, a message, a
+  child exiting). The test clock does not wait for it.
+"""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Callable
+from typing import Literal
+
+from pyfect._core import Effect, _Callback, _Sync
+from pyfect._wake_tracking import WakeTracker
+from pyfect.duration import Seconds
+from pyfect.scheduler import CurrentScheduler, Scheduler
+
+type Resume[A, E] = Callable[[Effect[A, E]], None]
+type Cancel = Callable[[], object]
+type Register[A, E] = Callable[[Resume[A, E]], Cancel | None]
+type WaitsFor = Literal["work", "event"]
+
+_KEEPALIVE = Seconds(0.25)
+"""How often a waiting fiber's timer fires; below the test runtime's
+one-second stall timeout."""
+
+
+class _Waiting:
+    __slots__ = ("active", "counted", "registering")
+
+    def __init__(self, *, counted: bool) -> None:
+        self.active = True
+        # Whether the scheduler's outside work still counts this wait; only
+        # read and written on the scheduler's thread.
+        self.counted = counted
+        # The thread running ``register``, while it runs: a resume from it
+        # happens at once.
+        self.registering: int | None = threading.get_ident()
+
+    def uncount(self, outside: WakeTracker) -> None:
+        if self.counted:
+            self.counted = False
+            outside.finish_one()
+
+
+def os_wait[A, E](register: Register[A, E], *, waits_for: WaitsFor) -> Effect[A, E]:
+    """Suspend until ``register``'s ``resume`` is called, from any thread.
+    ``register`` runs on the fiber's thread and may resume at once; the
+    cancel function it returns runs if the fiber is interrupted first.
+    ``waits_for`` says whether the outside is work this wait started or an
+    event that may never come (see the module)."""
+    return CurrentScheduler().flat_map(
+        lambda scheduler: _Callback(_registered(scheduler, register, waits_for))
+    )
+
+
+def _registered[A, E](
+    scheduler: Scheduler, register: Register[A, E], waits_for: WaitsFor
+) -> Callable[[Resume[A, E]], Effect[None] | None]:
+    def start(resume: Resume[A, E]) -> Effect[None] | None:
+        outside = scheduler.outside_work()
+        waiting = _Waiting(counted=waits_for == "work")
+        if waiting.counted:
+            outside.pending += 1
+
+        def beat() -> None:
+            if waiting.active:
+                scheduler.call_later(_KEEPALIVE, beat)
+
+        def settle(effect: Effect[A, E]) -> None:
+            # On the scheduler's thread: the resume and what it schedules
+            # count as outside work until the fiber suspends again.
+            if waiting.counted:
+                scheduler.track(outside, lambda: resume(effect))
+                waiting.uncount(outside)
+            else:
+                resume(effect)
+
+        def finish(effect: Effect[A, E]) -> None:
+            waiting.active = False
+            if waits_for == "event":
+                resume(effect)
+            elif waiting.registering == threading.get_ident():
+                resume(effect)
+                waiting.uncount(outside)
+            else:
+                scheduler.schedule(lambda: settle(effect))
+
+        cancel = register(finish)
+        waiting.registering = None
+        if not waiting.active:
+            return None
+        scheduler.call_later(_KEEPALIVE, beat)
+
+        def interrupted() -> None:
+            waiting.active = False
+            waiting.uncount(outside)
+            if cancel is not None:
+                cancel()
+
+        return _Sync(interrupted)
+
+    return start

pyfect/_path_posix.py

@@ -0,0 +1,129 @@
+"""Node's ``path.posix`` algorithms, as Effect's ``Path`` has them."""
+
+from __future__ import annotations
+
+import os
+
+
+def _segments(path: str, *, above_root: bool) -> str:
+    out: list[str] = []
+    for segment in path.split("/"):
+        if segment in {"", "."}:
+            continue
+        if segment != "..":
+            out.append(segment)
+        elif out and out[-1] != "..":
+            out.pop()
+        elif above_root:
+            out.append("..")
+    return "/".join(out)
+
+
+def normalize(path: str) -> str:
+    if not path:
+        return "."
+    absolute = path[0] == "/"
+    trailing = path[-1] == "/"
+    body = _segments(path, above_root=not absolute)
+    if not body:
+        return "/" if absolute else ("./" if trailing else ".")
+    if trailing:
+        body += "/"
+    return "/" + body if absolute else body
+
+
+def join(*paths: str) -> str:
+    parts = [part for part in paths if part]
+    return normalize("/".join(parts)) if parts else "."
+
+
+def resolve(*segments: str) -> str:
+    resolved = ""
+    for segment in reversed(segments):
+        if segment:
+            resolved = f"{segment}/{resolved}"
+            if segment[0] == "/":
+                break
+    else:
+        resolved = f"{os.getcwd()}/{resolved}"
+    return "/" + _segments(resolved, above_root=False)
+
+
+def relative(from_: str, to: str) -> str:
+    start = [part for part in resolve(from_).split("/") if part]
+    end = [part for part in resolve(to).split("/") if part]
+    common = 0
+    while common < min(len(start), len(end)) and start[common] == end[common]:
+        common += 1
+    return "/".join([*[".."] * (len(start) - common), *end[common:]])
+
+
+def _last_segment(path: str) -> str:
+    return path.rstrip("/").rpartition("/")[2]
+
+
+def basename(path: str, suffix: str = "") -> str:
+    if suffix and path == suffix:
+        return ""
+    base = _last_segment(path)
+    if suffix and base != suffix and base.endswith(suffix):
+        return base[: -len(suffix)]
+    return base
+
+
+def dirname(path: str) -> str:
+    if not path:
+        return "."
+    rooted = path[0] == "/"
+    stripped = path.rstrip("/")
+    end = stripped.rfind("/", 1)
+    if end == -1:
+        return "/" if rooted else "."
+    if rooted and end == 1:
+        return "//"
+    return stripped[:end]
+
+
+def extname(path: str) -> str:
+    base = _last_segment(path)
+    dot = base.rfind(".")
+    return "" if dot <= 0 or base == ".." else base[dot:]
+
+
+def parse(path: str) -> tuple[str, str, str, str, str]:
+    """Node's ``path.posix.parse``: root, dir, base, ext and name. Its own scan, as its check
+    for a last segment of ``..`` misses one right below the root (``/..`` has the ext ``.``)."""
+    if not path:
+        return "", "", "", "", ""
+    absolute = path.startswith("/")
+    first = 1 if absolute else 0
+    start_dot = end = -1
+    start_part = 0
+    before_dot = 0  # 0: nothing seen before the dot, 1: more dots, -1: another character
+    for i in range(len(path) - 1, first - 1, -1):
+        char = path[i]
+        if char == "/":
+            if end != -1:
+                start_part = i + 1
+                break
+            continue
+        if end == -1:
+            end = i + 1
+        if char == ".":
+            if start_dot == -1:
+                start_dot = i
+            elif before_dot != 1:
+                before_dot = 1
+        elif start_dot != -1:
+            before_dot = -1
+    base = ext = name = ""
+    if end != -1:
+        begin = 1 if start_part == 0 and absolute else start_part
+        base = path[begin:end]
+        is_dots = before_dot == 1 and start_dot == end - 1 and start_dot == start_part + 1
+        if start_dot == -1 or before_dot == 0 or is_dots:
+            name = base
+        else:
+            name, ext = path[begin:start_dot], path[start_dot:end]
+    directory = path[: start_part - 1] if start_part > 0 else "/" if absolute else ""
+    return "/" if absolute else "", directory, base, ext, name