pyfect-dev 0.1.0__py3-none-any.whl

pyfect/otel/tracer.py

@@ -0,0 +1,157 @@
+"""pyfect tracing into OpenTelemetry (Effect's ``OtelTracer``).
+
+The tracer this module installs makes each pyfect span an OpenTelemetry
+span (``OtelSpan``) and keeps the current span active in OpenTelemetry's
+context while a fiber runs, so libraries instrumented with OpenTelemetry
+see it as their parent::
+
+    tracing = otel.tracer.layer.pipe(layer.provide(provider_layer))
+    program.with_span("request").pipe(fx.provide(tracing))
+
+A pyfect parent wins over the active OpenTelemetry span; with no pyfect
+parent, a span started by other OpenTelemetry code is the parent.
+``make_external_span`` and ``with_span_context`` continue a remote trace.
+Nothing is exported unless the tracer provider has span processors (see
+``otel.sdk``).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any, Final, Never
+
+from opentelemetry import context as otel_context
+from opentelemetry import trace
+from opentelemetry.trace import SpanContext
+
+from pyfect import layer as _layer
+from pyfect._core import Effect
+from pyfect._fiber import Fiber
+from pyfect._tracer_ref import CurrentTracer
+from pyfect.clock import Clock, ClockService
+from pyfect.context import Tag
+from pyfect.errors import NoSuchElementException
+from pyfect.layer import Layer
+from pyfect.otel._attributes import PYFECT_VERSION
+from pyfect.otel._span import (
+    EFFECT_SPAN_KEY,
+    OtelSpan,
+    OtelTraceFlags,
+    OtelTraceState,
+    external_span_of,
+    make_external_span,
+    populate_context,
+    propagated_span,
+)
+from pyfect.otel._wrapper import EffectSpanView
+from pyfect.tracer import ParentSpan, Span, Tracer, current_span
+
+__all__ = [
+    "EffectSpanView",
+    "NoSuchElementException",
+    "OtelSpan",
+    "OtelTraceFlags",
+    "OtelTraceState",
+    "OtelTracer",
+    "OtelTracerProvider",
+    "current_otel_span",
+    "layer",
+    "layer_global",
+    "layer_global_provider",
+    "layer_global_tracer",
+    "layer_tracer",
+    "layer_without_otel_tracer",
+    "make",
+    "make_external_span",
+    "with_span_context",
+]
+
+
+class OtelTracer(Tag[trace.Tracer]):
+    """The OpenTelemetry tracer pyfect spans are started with."""
+
+
+class OtelTracerProvider(Tag[trace.TracerProvider]):
+    """The OpenTelemetry tracer provider ``layer_tracer`` gets a tracer
+    from."""
+
+
+def _context_hook[T](run: Callable[[], T], fiber: Fiber[Any, Any], /) -> T:
+    """Runs a fiber step with the fiber's current span active in
+    OpenTelemetry's context."""
+    span = propagated_span(fiber.get_ref(ParentSpan))
+    if span is None:
+        return run()
+    context = populate_context(otel_context.get_current(), span)
+    context = otel_context.set_value(EFFECT_SPAN_KEY, trace.get_current_span(context), context)
+    token = otel_context.attach(context)
+    try:
+        return run()
+    finally:
+        otel_context.detach(token)
+
+
+def _tracer(otel_tracer: trace.Tracer) -> Tracer:
+    return Tracer(lambda request: OtelSpan(otel_tracer, request), _context_hook)
+
+
+make: Final[Effect[Tracer, Never, OtelTracer]] = OtelTracer().map(_tracer)
+"""A pyfect tracer starting OpenTelemetry spans with the ``OtelTracer``."""
+
+layer_global_provider: Final[Layer[OtelTracerProvider]] = _layer.sync(
+    OtelTracerProvider, trace.get_tracer_provider
+)
+"""The global OpenTelemetry tracer provider."""
+
+layer_tracer: Final[Layer[OtelTracer, Never, OtelTracerProvider]] = _layer.effect(
+    OtelTracer, OtelTracerProvider().map(lambda p: p.get_tracer("pyfect", PYFECT_VERSION))
+)
+"""A tracer from the provider, with ``pyfect`` and its version as the
+instrumentation scope."""
+
+layer_global_tracer: Final[Layer[OtelTracer]] = layer_tracer.pipe(
+    _layer.provide(layer_global_provider)
+)
+"""A tracer from the global tracer provider."""
+
+layer_without_otel_tracer: Final[Layer[Never, Never, OtelTracer]] = _layer.effect(
+    CurrentTracer, make
+)
+"""Installs the bridge's tracer, using the ``OtelTracer`` provided."""
+
+layer: Final[Layer[OtelTracer, Never, OtelTracerProvider]] = layer_without_otel_tracer.pipe(
+    _layer.provide_merge(layer_tracer)
+)
+"""Installs the bridge's tracer, with a tracer from the provider."""
+
+layer_global: Final[Layer[OtelTracer]] = layer_without_otel_tracer.pipe(
+    _layer.provide_merge(layer_global_tracer)
+)
+"""Installs the bridge's tracer, with a tracer from the global provider."""
+
+
+def _otel_span(span: Span, clock: ClockService) -> trace.Span:
+    return span.span if isinstance(span, OtelSpan) else EffectSpanView(span, clock)
+
+
+def current_otel_span() -> Effect[trace.Span, NoSuchElementException]:
+    """The current span as an OpenTelemetry span: the bridge's own, or a
+    view writing through to a span another tracer made. Fails outside every
+    span, or in an external one."""
+    return Clock().flat_map(lambda clock: current_span().map(lambda s: _otel_span(s, clock)))
+
+
+class _WithSpanContext:
+    __slots__ = ("_span_context",)
+
+    def __init__(self, span_context: SpanContext) -> None:
+        self._span_context = span_context
+
+    def __call__[A, E, R](self, eff: Effect[A, E, R], /) -> Effect[A, E, R]:
+        return eff.with_parent_span(external_span_of(self._span_context))
+
+
+def with_span_context(span_context: SpanContext) -> _WithSpanContext:
+    """Run an effect with the OpenTelemetry span context as its parent span
+    (use with ``.pipe``), continuing a trace begun outside pyfect."""
+    return _WithSpanContext(span_context)

pyfect/partitioned_semaphore.py

@@ -0,0 +1,211 @@
+"""``PartitionedSemaphore[K]``: one pool of permits shared fairly by keys.
+
+Waiters are grouped by key, and freed permits go round-robin across keys,
+one permit per turn, so a key with many waiters cannot starve the others. A
+waiter that needs several permits collects them as they come and is woken
+once it has them all. Only the wait is interruptible: an interrupt at any
+point before the permits are guarded gives back what the waiter collected.
+
+A request for more permits than the capacity never completes, as in Effect.
+"""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+from collections.abc import Callable, Hashable
+from typing import TYPE_CHECKING, Any
+
+from pyfect._core import (
+    Effect,
+    Frame,
+    Restore,
+    Step,
+    _Callback,
+    _Succeed,
+    _Suspend,
+    _Sync,
+    _WithFiber,
+    uninterruptible_mask,
+)
+from pyfect._effect_args import require_effect
+from pyfect._round_robin import RoundRobin
+from pyfect.cause import Cause
+from pyfect.effect import never
+from pyfect.exit import Failure
+from pyfect.option import Option, Some
+
+if TYPE_CHECKING:
+    from pyfect._fiber import Fiber
+
+_VOID: Effect[None] = _Succeed(None)
+
+
+class _Waiter:
+    __slots__ = ("missing", "resume")
+
+    def __init__(self, missing: int) -> None:
+        self.missing = missing
+        self.resume: Callable[[Effect[None]], None] | None = None
+
+    def register(self, resume: Callable[[Effect[None]], None]) -> None:
+        if self.missing == 0:  # handed its permits before it parked
+            resume(_VOID)
+        else:
+            self.resume = resume
+
+    def wake(self) -> None:
+        if self.resume is not None:
+            self.resume(_VOID)
+
+
+class _GiveBackOnFailure(Frame):
+    """Below a standalone ``take``: once it has taken its permits, give
+    them back if the take still fails (an interrupt landing as it returns),
+    as Effect's ``take`` does."""
+
+    __slots__ = ("armed", "release")
+    # Right below the claim: the permits reach the caller with no pause.
+    claims = True
+
+    def __init__(self, release: Callable[[], object]) -> None:
+        self.release = release
+        self.armed = False
+
+    def _on_failure(self, cause: Cause[Any], fiber: Fiber[Any, Any]) -> Step:
+        if self.armed:
+            self.release()
+        return Failure(cause)
+
+
+class PartitionedSemaphore[K: Hashable = Hashable]:
+    __slots__ = ("_pending", "_releasing", "_total", "_waiters", "_waiting", "capacity")
+
+    def __init__(self, permits: int) -> None:
+        self.capacity = max(0, permits)
+        self._total = self.capacity  # free permits nobody is waiting for
+        self._waiting = 0  # permits waiters still miss
+        self._pending = 0  # released permits not yet handed out
+        self._releasing = False
+        # Typed by Hashable, not K, so K only goes in (contravariant, as in Effect).
+        self._waiters: RoundRobin[Hashable, _Waiter] = RoundRobin()
+
+    @classmethod
+    def make(cls, permits: int) -> Effect[PartitionedSemaphore[K]]:
+        """``PartitionedSemaphore[str].make(4)``"""
+        return _Sync(lambda: cls(permits))
+
+    @property
+    def available(self) -> Effect[int]:
+        return _Sync(lambda: self._total)
+
+    def take(self, key: K, n: int = 1) -> Effect[None]:
+        """Wait in ``key``'s queue for ``n`` permits and take them. Prefer
+        ``with_permits``, which also gives them back."""
+        if n <= 0:
+            return _VOID
+        if self.capacity < n:
+            return never
+
+        def run(fiber: Fiber[Any, Any]) -> Effect[None]:
+            guard = _GiveBackOnFailure(lambda: self._release(n))
+            fiber._push(guard)
+
+            def arm(_: None) -> None:
+                guard.armed = True
+
+            return uninterruptible_mask(lambda restore: self._acquire(key, n, restore).map(arm))
+
+        return _WithFiber(run)
+
+    def release(self, n: int = 1) -> Effect[int]:
+        """Give back ``n`` permits; succeeds with the number now free."""
+        return _Sync(lambda: self._release(n))
+
+    def with_permits[A, E, R](self, key: K, eff: Effect[A, E, R], n: int = 1) -> Effect[A, E, R]:
+        """Run ``eff`` holding ``n`` permits taken in ``key``'s queue,
+        releasing them however it ends."""
+        require_effect(eff, "eff")
+        if n <= 0:
+            return eff
+        if self.capacity < n:
+            return never
+        release = self.release(n)
+        return uninterruptible_mask(
+            lambda restore: self._acquire(key, n, restore).and_then(restore(eff).ensuring(release))
+        )
+
+    def with_permits_if_available[A, E, R](
+        self, eff: Effect[A, E, R], n: int = 1
+    ) -> Effect[Option[A], E, R]:
+        """Run ``eff`` holding ``n`` permits if they are free now (``Some``
+        of its result); otherwise ``None`` without running it."""
+        require_effect(eff, "eff")
+        if n <= 0:
+            return eff.map(Some)
+
+        def body(restore: Restore) -> Effect[Option[A], E, R]:
+            def run() -> Effect[Option[A], E, R]:
+                if self._total < n:
+                    return _Succeed(None)
+                self._total -= n
+                return restore(eff.map(Some)).ensuring(self.release(n))
+
+            return _Suspend(run)
+
+        return uninterruptible_mask(body)
+
+    def _acquire(self, key: K, n: int, restore: Restore) -> Effect[None]:
+        """Take ``n`` permits; runs uninterruptibly, with only the wait
+        restored. Once it succeeds the caller holds them."""
+
+        def run() -> Effect[None]:
+            if self._total >= n:
+                self._total -= n
+                return _VOID
+            waiter = _Waiter(n - self._total)
+            self._total = 0
+            self._waiting += waiter.missing
+            self._waiters.add(key, waiter)
+            return restore(_Callback(waiter.register)).on_error(
+                lambda _: _Sync(lambda: self._give_up(key, n, waiter))
+            )
+
+        return _Suspend(run)
+
+    def _give_up(self, key: K, n: int, waiter: _Waiter) -> None:
+        self._waiters.remove(key, waiter)
+        self._waiting -= waiter.missing
+        self._release(n - waiter.missing)
+
+    def _release(self, n: int) -> int:
+        if n > 0:
+            self._pending += n
+        # A woken waiter can release again from inside this loop; the
+        # running loop hands those permits out instead of recursing.
+        if not self._releasing:
+            self._releasing = True
+            try:
+                self._hand_out()
+            finally:
+                self._releasing = False
+        # Free permits must be visible before a reentrant release returns.
+        if self._waiting == 0:
+            self._total = min(self.capacity, self._total + self._pending)
+            self._pending = 0
+        return self._total
+
+    def _hand_out(self) -> None:
+        while self._pending > 0 and self._waiting > 0:
+            found = self._waiters.next()
+            if found is None:
+                return
+            key, waiter = found
+            waiter.missing -= 1
+            self._waiting -= 1
+            self._pending -= 1
+            if waiter.missing == 0:
+                self._waiters.remove(key, waiter)
+                waiter.wake()
+
+
+__all__ = ["PartitionedSemaphore"]

pyfect/path.py

@@ -0,0 +1,304 @@
+"""Path operations as a service (Effect's ``Path``).
+
+Programs that take the ``Path`` service instead of importing ``os.path``
+can be run against another platform's rules (``layer_posix``,
+``layer_win32``) or a test double::
+
+    p = yield from Path()
+    config = p.join(home, ".config", "app.toml")
+
+``posix`` follows Node's ``path.posix``, as Effect does (``join("/a", "/b")``
+is ``"/a/b"``, ``dirname("/a/b/")`` is ``"/a"``, ``relative`` of a path to
+itself is ``""``). ``win32`` is ``ntpath``.
+"""
+
+from __future__ import annotations
+
+import ntpath
+import os
+import posixpath
+import re
+import urllib.parse
+from dataclasses import dataclass
+from typing import Never, Protocol
+
+from pyfect import _path_posix
+from pyfect._core import Effect
+from pyfect.context import Tag
+from pyfect.effect import fail, succeed
+from pyfect.layer import Layer
+from pyfect.layer import succeed as layer_succeed
+from pyfect.platform_error import BadArgument
+
+
+class _PathModule(Protocol):
+    """The part of ``posixpath`` / ``ntpath`` the service uses."""
+
+    @property
+    def sep(self) -> str: ...
+    def basename(self, p: str, /) -> str: ...
+    def dirname(self, p: str, /) -> str: ...
+    def splitext(self, p: str, /) -> tuple[str, str]: ...
+    def splitroot(self, p: str, /) -> tuple[str, str, str]: ...
+    def isabs(self, s: str, /) -> bool: ...
+    def join(self, a: str, /, *paths: str) -> str: ...
+    def normpath(self, path: str, /) -> str: ...
+    def relpath(self, path: str, start: str | None = None) -> str: ...
+    def abspath(self, path: str, /) -> str: ...
+
+
+@dataclass(frozen=True, slots=True)
+class Parsed:
+    """The parts of a path (Node's ``path.parse``): ``/home/u/f.txt`` has
+    root ``"/"``, dir ``"/home/u"``, base ``"f.txt"``, name ``"f"`` and ext
+    ``".txt"``."""
+
+    root: str = ""
+    dir: str = ""
+    base: str = ""
+    ext: str = ""
+    name: str = ""
+
+
+_ENCODED_SEPARATOR = re.compile(r"%2f", re.IGNORECASE)
+_ENCODED_BACKSLASH = re.compile(r"%5c", re.IGNORECASE)
+_BAD_ESCAPE = re.compile(r"%(?![0-9A-Fa-f]{2})")
+# WHATWG's path percent-encode set leaves these as they are.
+_URL_SAFE = "/!$&'()*+,;=:@-._~"
+
+
+class PathService:
+    """``Path`` over one ``os.path`` module (``posixpath`` or ``ntpath``)."""
+
+    __slots__ = ("_m", "_windows")
+
+    def __init__(self, module: _PathModule, *, windows: bool) -> None:
+        self._m = module
+        self._windows = windows
+
+    @property
+    def sep(self) -> str:
+        return self._m.sep
+
+    def basename(self, path: str, suffix: str = "") -> str:
+        """The last part of ``path``, without ``suffix`` when it ends with
+        it (and is more than it)."""
+        base = self._m.basename(path)
+        if suffix and base.endswith(suffix) and base != suffix:
+            return base[: -len(suffix)]
+        return base
+
+    def dirname(self, path: str) -> str:
+        return self._m.dirname(path)
+
+    def extname(self, path: str) -> str:
+        """The extension of the last part, dot included (``""`` for
+        ``.bashrc``)."""
+        return self._m.splitext(path)[1]
+
+    def format(
+        self, *, root: str = "", dir: str = "", base: str = "", name: str = "", ext: str = ""
+    ) -> str:
+        """Node's ``path.format``: ``dir`` (else ``root``), then ``base``
+        (else ``name + ext``)."""
+        directory = dir or root
+        last = base or name + ext
+        if not directory:
+            return last
+        if directory == root:
+            return directory + last
+        return directory + self.sep + last
+
+    def from_file_url(self, url: str) -> Effect[str, BadArgument]:
+        """The path a ``file:`` URL names. Fails for another scheme, a host
+        (but a Windows share), an encoded separator or a broken escape."""
+        parts = urllib.parse.urlsplit(url)
+        if parts.scheme != "file":
+            return _bad_url("URL must be of scheme file")
+        host = parts.hostname or ""
+        if host == "localhost":
+            host = ""
+        if host and not self._windows:
+            return _bad_url("Invalid file URL host")
+        path = _remove_dot_segments(parts.path)
+        encoded = _ENCODED_SEPARATOR.search(path) or (
+            self._windows and _ENCODED_BACKSLASH.search(path)
+        )
+        if encoded:
+            return _bad_url("must not include encoded / characters")
+        try:
+            decoded = _decode_uri_component(path)
+        except ValueError as error:
+            return fail(BadArgument(module="Path", method="from_file_url", cause=error))
+        return succeed(_windows_path(host, decoded) if self._windows else decoded)
+
+    def is_absolute(self, path: str) -> bool:
+        return self._m.isabs(path)
+
+    def join(self, *paths: str) -> str:
+        return self._m.join(*paths) if paths else "."
+
+    def normalize(self, path: str) -> str:
+        return self._m.normpath(path)
+
+    def parse(self, path: str) -> Parsed:
+        drive, root, _tail = self._m.splitroot(path)
+        base = self._m.basename(path)
+        name, ext = self._m.splitext(base)
+        return Parsed(root=drive + root, dir=self._m.dirname(path), base=base, ext=ext, name=name)
+
+    def relative(self, from_: str, to: str) -> str:
+        """The path from ``from_`` to ``to`` (``os.path.relpath``); ``to``
+        made absolute when no relative path leads there (another drive)."""
+        try:
+            return self._m.relpath(to, from_)
+        except ValueError:
+            return self._m.abspath(to)
+
+    def resolve(self, *segments: str) -> str:
+        """The absolute path ``segments`` lead to from the working
+        directory."""
+        return self._m.abspath(self._m.join("", *segments))
+
+    def resolve_within(self, root: str, input: str) -> str | None:
+        """``input`` resolved below ``root``, or ``None`` when it is rooted
+        or leads outside ``root`` (a ``..`` past it)."""
+        if self.parse(input).root != "":
+            return None
+        base = self.resolve(root)
+        candidate = self.resolve(base, input)
+        relative = self.relative(base, candidate)
+        if relative == ".." or relative.startswith(".." + self.sep) or self.is_absolute(relative):
+            return None
+        return candidate
+
+    def to_file_url(self, path: str) -> Effect[str, BadArgument]:
+        """The ``file:`` URL of ``path``, made absolute first."""
+        resolved = self.resolve(path)
+        if self._windows:
+            resolved = resolved.replace("\\", "/")
+            if resolved.startswith("//"):
+                host, _, rest = resolved[2:].partition("/")
+                return succeed(f"file://{host}/{urllib.parse.quote(rest, safe=_URL_SAFE)}")
+            resolved = "/" + resolved
+        elif path.endswith("/") and not resolved.endswith("/"):
+            resolved += "/"
+        return succeed("file://" + urllib.parse.quote(resolved, safe=_URL_SAFE))
+
+    def to_namespaced_path(self, path: str) -> str:
+        """Windows' ``\\\\?\\`` form of an absolute path; on POSIX, ``path``."""
+        if not self._windows or not path:
+            return path
+        resolved = self.resolve(path)
+        if resolved.startswith("\\\\?\\"):
+            return resolved
+        if resolved.startswith("\\\\"):
+            return "\\\\?\\UNC\\" + resolved[2:]
+        if re.match(r"[A-Za-z]:\\", resolved):
+            return "\\\\?\\" + resolved
+        return path
+
+
+_SINGLE_DOT = frozenset((".", "%2e"))
+_DOUBLE_DOT = frozenset(("..", ".%2e", "%2e.", "%2e%2e"))
+_DRIVE = re.compile(r"[A-Za-z][:|]")
+
+
+def _remove_dot_segments(path: str) -> str:
+    """The WHATWG URL parser's handling of ``.`` and ``..`` segments (encoded
+    dots included) in a ``file:`` URL's path; ``..`` never leaves a drive
+    letter."""
+    segments = path.split("/")[1:]
+    kept: list[str] = []
+    for index, segment in enumerate(segments):
+        last = index == len(segments) - 1
+        dots = segment.lower()
+        if dots in _DOUBLE_DOT:
+            if kept and not (len(kept) == 1 and _DRIVE.fullmatch(kept[0])):
+                kept.pop()
+            if last:
+                kept.append("")
+        elif dots in _SINGLE_DOT:
+            if last:
+                kept.append("")
+        else:
+            kept.append(segment)
+    return "/" + "/".join(kept) if segments else path
+
+
+def _windows_path(host: str, decoded: str) -> str:
+    path = decoded.replace("/", "\\")
+    if host:
+        return f"\\\\{host}{path}"
+    return path[1:] if re.match(r"\\[A-Za-z]:", path) else path
+
+
+def _bad_url(description: str) -> Effect[Never, BadArgument]:
+    return fail(BadArgument(module="Path", method="from_file_url", description=description))
+
+
+def _decode_uri_component(text: str) -> str:
+    """JavaScript's ``decodeURIComponent``: a ``%`` without two hex digits,
+    or escapes that are not UTF-8, raise ``ValueError``."""
+    if _BAD_ESCAPE.search(text):
+        raise ValueError(f"URI malformed: {text}")
+    return urllib.parse.unquote_to_bytes(text).decode("utf-8", errors="strict")
+
+
+class Path(Tag[PathService]):
+    """The path rules a program uses."""
+
+
+class _NodePosix(PathService):
+    """The operations whose results differ between Node and ``posixpath``."""
+
+    __slots__ = ()
+
+    def basename(self, path: str, suffix: str = "") -> str:
+        return _path_posix.basename(path, suffix)
+
+    def dirname(self, path: str) -> str:
+        return _path_posix.dirname(path)
+
+    def extname(self, path: str) -> str:
+        return _path_posix.extname(path)
+
+    def join(self, *paths: str) -> str:
+        return _path_posix.join(*paths)
+
+    def normalize(self, path: str) -> str:
+        return _path_posix.normalize(path)
+
+    def parse(self, path: str) -> Parsed:
+        root, directory, base, ext, name = _path_posix.parse(path)
+        return Parsed(root=root, dir=directory, base=base, ext=ext, name=name)
+
+    def relative(self, from_: str, to: str) -> str:
+        return _path_posix.relative(from_, to)
+
+    def resolve(self, *segments: str) -> str:
+        return _path_posix.resolve(*segments)
+
+
+posix: PathService = _NodePosix(posixpath, windows=False)
+"""POSIX rules (Node's ``path.posix``)."""
+
+win32 = PathService(ntpath, windows=True)
+"""Windows rules (``ntpath``)."""
+
+layer_posix: Layer[Path] = layer_succeed(Path, posix)
+layer_win32: Layer[Path] = layer_succeed(Path, win32)
+layer: Layer[Path] = layer_win32 if os.name == "nt" else layer_posix
+"""The host's rules."""
+
+__all__ = [
+    "BadArgument",
+    "Parsed",
+    "Path",
+    "PathService",
+    "layer",
+    "layer_posix",
+    "layer_win32",
+    "posix",
+    "win32",
+]