pyfect-dev 0.1.0__py3-none-any.whl

pyfect/http/httpx_client.py

@@ -0,0 +1,291 @@
+"""An ``HttpClient`` backed by httpx (the ``pyfect-dev[httpx]`` extra).
+
+``layer`` provides the client; ``layer_with(**options)`` passes options to
+``httpx.AsyncClient`` (``verify``, ``proxy``, ``limits``, ...). The httpx
+client runs on an asyncio loop of its own, on a background thread, so the
+layer works under ``run_blocking`` and ``run_async`` alike, and closes with the
+layer::
+
+    program.pipe(fx.provide(httpx_client.layer))
+
+Interrupting a request cancels it in httpx (closing its connection);
+response bodies stream chunk by chunk, and a request body ``Stream`` is
+pulled as httpx sends it. httpx applies no timeout by default here, as
+``fetch`` applies none: use ``Effect.timeout``.
+"""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncIterator, Callable, Coroutine
+from concurrent.futures import Future
+from typing import TYPE_CHECKING, Any, Never
+from urllib.parse import SplitResult
+
+from pyfect._fiber import Fiber
+from pyfect._js import text_encode
+from pyfect._loop_thread import LoopThread
+from pyfect._os_wait import WaitsFor, os_wait
+from pyfect._released import Lifetime
+from pyfect.effect import Effect, fail, succeed, sync
+from pyfect.errors import Done
+from pyfect.http import body as _body
+from pyfect.http import url as _url
+from pyfect.http._httpx_options import RequestDefaults, import_httpx, split_options
+from pyfect.http.abort import AbortSignal
+from pyfect.http.client import HttpClient, HttpClientService
+from pyfect.http.client import make as make_client
+from pyfect.http.error import EncodeError, HttpClientError, TransportError
+from pyfect.http.headers import Headers
+from pyfect.http.method import has_body
+from pyfect.http.request import HttpClientRequest
+from pyfect.http.response import HttpClientResponse
+from pyfect.layer import Layer
+from pyfect.layer import effect as layer_effect
+from pyfect.scope import Scope, acquire_release
+from pyfect.stream import Stream, from_effect_repeat
+
+if TYPE_CHECKING:
+    import httpx
+
+_NO_BODY_STATUSES = frozenset((204, 205, 304))
+_EMPTY_FORM_BOUNDARY = "pyfect-empty-form"
+
+
+def _form_content(form: _body.FormData) -> dict[str, Any]:
+    if not form.entries:
+        # httpx sends an empty ``files`` list as no body at all.
+        return {"content": text_encode(f"--{_EMPTY_FORM_BOUNDARY}--\r\n")}
+    files = [
+        (
+            name,
+            (None, text_encode(value))
+            if isinstance(value, str)
+            else (value.name, value.content, value.content_type),
+        )
+        for name, value in form.entries
+    ]
+    return {"files": files}
+
+
+def _future[A](
+    signal: AbortSignal | None, start: Callable[[], Future[A]], *, waits_for: WaitsFor
+) -> Effect[A, BaseException]:
+    """Wait for the future ``start()`` gives; interrupting (or aborting
+    ``signal``) cancels it. A body's next chunk is an ``"event"``: the
+    server may hold it back indefinitely, so the test clock must not wait
+    for it."""
+
+    def register(resume: Callable[[Effect[A, BaseException]], None]) -> Callable[[], bool]:
+        future = start()
+        remove = (lambda: None) if signal is None else signal.add_listener(future.cancel)
+
+        def done(finished: Future[A]) -> None:
+            remove()
+            if finished.cancelled():
+                resume(fail(asyncio.CancelledError()))
+                return
+            error = finished.exception()
+            resume(succeed(finished.result()) if error is None else fail(error))
+
+        future.add_done_callback(done)
+        return future.cancel
+
+    return os_wait(register, waits_for=waits_for)
+
+
+class _Backend:
+    """An ``httpx.AsyncClient`` on an asyncio loop running on a thread of
+    its own."""
+
+    def __init__(self, options: dict[str, Any]) -> None:
+        options.setdefault("timeout", None)
+        self.client = import_httpx().AsyncClient(**options)
+        self._thread = LoopThread("pyfect-httpx")
+        self.loop = self._thread.loop
+        self._lifetime = Lifetime("HTTP client")
+        self._closing: set[asyncio.Task[None]] = set()
+        """Responses being closed (touched on the loop only)."""
+
+    def _close_response(self, response: httpx.Response) -> None:
+        task = self.loop.create_task(response.aclose())
+        self._closing.add(task)
+        task.add_done_callback(self._closing.discard)
+
+    async def _settle(self) -> None:
+        """Wait for the responses being closed."""
+        while self._closing:
+            closing = list(self._closing)
+            self._closing.clear()
+            await asyncio.gather(*closing, return_exceptions=True)
+
+    async def _shutdown(self) -> None:
+        await self._settle()
+        await self.client.aclose()
+
+    def submit[A](self, make: Callable[[], Coroutine[Any, Any, A]]) -> Future[A]:
+        """Run ``make()`` on the loop; once the client is closed, a future
+        failed with ``Released``."""
+        return self._lifetime.submit(lambda: self._thread.submit(make))
+
+    def _call_soon(self, f: Callable[[], object]) -> None:
+        """Run ``f`` on the loop; once the backend has closed (and with it
+        every response), there is nothing left to do."""
+        try:
+            self.loop.call_soon_threadsafe(f)
+        except RuntimeError:
+            if not self.loop.is_closed():
+                raise
+
+    def close(self) -> Effect[None]:
+        """Close the httpx client, then stop the loop and wait until it has
+        closed (with it every response body httpx left open)."""
+
+        def shutdown() -> Future[None]:
+            future = self.submit(self._shutdown)
+            self._lifetime.close()
+            return future
+
+        def stop() -> Future[None]:
+            self._thread.stop()
+            return self._thread.stopped
+
+        closing = _future(None, shutdown, waits_for="work").ignore()
+        return closing.and_then(_future(None, stop, waits_for="work").ignore())
+
+    def _content(
+        self, request: HttpClientRequest, fiber: Fiber[Any, Any], failures: list[object]
+    ) -> dict[str, Any]:
+        body = request.body
+        if not has_body(request.method):
+            return {}
+        match body:
+            case _body.Bytes():
+                return {"content": body.body}
+            case _body.Raw():
+                return {"content": body.body}
+            case _body.FormData():
+                return _form_content(body)
+            case _body.Stream():
+                chunks = body.stream.tap_error(lambda error: sync(lambda: failures.append(error)))
+                iterable = chunks.to_async_iterable_with(fiber._context)
+
+                async def content() -> AsyncIterator[bytes]:
+                    async for chunk in iterable:
+                        yield chunk
+
+                return {"content": content()}
+            case _body.Empty():
+                return {}
+
+    def send(
+        self,
+        request: HttpClientRequest,
+        url: SplitResult,
+        signal: AbortSignal,
+        fiber: Fiber[Any, Any],
+    ) -> Effect[HttpClientResponse, HttpClientError]:
+        if "upgrade" in request.headers:
+            return fail(
+                HttpClientError(
+                    TransportError(request, None, "httpx cannot send an HTTP upgrade request")
+                )
+            )
+        failures: list[object] = []
+        content = self._content(request, fiber, failures)
+        headers = request.headers
+        if isinstance(request.body, _body.FormData):
+            headers = headers.remove("content-type")
+            if not request.body.entries:
+                content_type = f"multipart/form-data; boundary={_EMPTY_FORM_BOUNDARY}"
+                headers = headers.set("content-type", content_type)
+
+        async def send() -> httpx.Response:
+            built = self.client.build_request(
+                request.method, _url.href(url), headers=list(headers.items()), **content
+            )
+            response = await self.client.send(built, stream=True)
+            # Tied to the signal as soon as it exists: an interrupt that wins
+            # against its arrival (a lost race) drops it unseen, and aborts.
+            signal.add_listener(lambda: self._call_soon(lambda: self._close_response(response)))
+            return response
+
+        def on_error(error: BaseException) -> Effect[Never, HttpClientError]:
+            if failures:
+                return fail(HttpClientError(EncodeError(request, failures[0])))
+            return fail(HttpClientError(TransportError(request, error)))
+
+        return (
+            _future(signal, lambda: self.submit(send), waits_for="work")
+            .catch_all(on_error)
+            .map(lambda response: self._response(request, response))
+        )
+
+    def _response(self, request: HttpClientRequest, response: httpx.Response) -> HttpClientResponse:
+        headers = Headers(
+            {name: ", ".join(response.headers.get_list(name)) for name in response.headers}
+        )
+        body: Stream[bytes, object] | None = None
+        if request.method != "HEAD" and response.status_code not in _NO_BODY_STATUSES:
+            body = self._body(response)
+        else:
+            self._call_soon(lambda: self._close_response(response))
+        return HttpClientResponse(
+            request,
+            response.status_code,
+            headers,
+            body,
+            url=str(response.url),
+            set_cookie=response.headers.get_list("set-cookie"),
+        )
+
+    def _body(self, response: httpx.Response) -> Stream[bytes, object]:
+        chunks = response.aiter_bytes()
+
+        async def next_chunk() -> bytes:
+            return await anext(chunks)
+
+        def end_or_fail(error: BaseException) -> Effect[Never, BaseException | Done]:
+            return fail(Done() if isinstance(error, StopAsyncIteration) else error)
+
+        pull = _future(None, lambda: self.submit(next_chunk), waits_for="event").catch_all(
+            end_or_fail
+        )
+        closing = _future(None, lambda: self.submit(response.aclose), waits_for="work").ignore()
+        return from_effect_repeat(pull).ensuring(closing)
+
+
+def make(**options: Any) -> Effect[HttpClientService[HttpClientError], Never, Scope]:
+    """A client on a new ``httpx.AsyncClient(**options)``, closed with the
+    scope.
+
+    The request defaults (``headers``, ``params``, ``cookies``, ``auth``)
+    are added to each request by the pyfect client, not by httpx, so
+    ``follow_redirects`` drops their credentials on a hop to another origin
+    as it does a request's own. ``auth`` is a ``(username, password)`` pair
+    and ``cookies`` a mapping; set any other credential per request."""
+    defaults, transport = split_options(options)
+    backend = acquire_release(
+        sync(lambda: _Backend(dict(transport))), lambda backend, _exit: backend.close()
+    )
+    return backend.map(lambda backend: _with_defaults(make_client(backend.send), defaults))
+
+
+def _with_defaults(
+    client: HttpClientService[HttpClientError], defaults: RequestDefaults
+) -> HttpClientService[HttpClientError]:
+    return client if defaults.is_empty() else client.map_request(defaults.apply)
+
+
+def layer_with(**options: Any) -> Layer[HttpClient]:
+    """``layer`` with options for ``httpx.AsyncClient`` (see ``make``)."""
+    split_options(options)
+    return layer_effect(HttpClient, make(**options))
+
+
+layer: Layer[HttpClient] = layer_with()
+"""An ``HttpClient`` on httpx with its default options (and no timeout)."""
+
+
+__all__ = ["layer", "layer_with", "make"]

pyfect/http/hypercorn_server.py

@@ -0,0 +1,122 @@
+"""An ``HttpServer`` on hypercorn (extra ``pyfect-dev[http2]``), which speaks
+HTTP/2 as Effect's ``NodeHttpServer`` does when given ``node:http2``'s server:
+cleartext (``h2c``) by default, with ALPN when ``config`` gives ``certfile`` and
+``keyfile``::
+
+    main = server.serve(app).pipe(layer.provide(hypercorn_server.layer(port=8080)))
+
+Otherwise as ``uvicorn_server``, on the same request handling and shutdown.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import socket
+from collections.abc import Mapping
+from typing import Any, NotRequired, Unpack, cast
+
+from pyfect._extras import missing_extra
+
+try:
+    from hypercorn.app_wrappers import ASGIWrapper
+    from hypercorn.asyncio.run import worker_serve  # pyright: ignore[reportUnknownVariableType]
+    from hypercorn.config import Config, Sockets
+except ImportError as error:  # pragma: no cover - depends on the environment
+    raise missing_extra("pyfect.http.hypercorn_server", "http2") from error
+
+from pyfect._core import Effect
+from pyfect.file_system import FileSystem
+from pyfect.http._asgi_listener import Dispatch, ListenOptions, graceful, make_listener
+from pyfect.http.client import HttpClient
+from pyfect.http.etag import Generator
+from pyfect.http.httpx_client import layer as httpx_layer
+from pyfect.http.live_server import layer_services
+from pyfect.http.live_server._listener import ABORT_GRACE_PERIOD
+from pyfect.http.platform import HttpPlatform
+from pyfect.http.server import HttpServer, HttpServerService, layer_test_client
+from pyfect.layer import Layer, merge, provide_merge
+from pyfect.layer import effect as layer_effect
+from pyfect.path import Path
+from pyfect.scope import Scope as ScopeTag
+from pyfect.socket.server import SocketServerError
+
+
+class Options(ListenOptions):
+    config: NotRequired[Mapping[str, Any]]
+    """More ``hypercorn.Config`` attributes (``certfile``, ``keyfile``,
+    ``alpn_protocols``, ``h2_max_concurrent_streams``...)."""
+
+
+class _Hypercorn:
+    """hypercorn as an ``Engine``: it takes its sockets and a trigger to stop."""
+
+    def __init__(self, dispatch: Dispatch, options: Options) -> None:
+        self.dispatch = dispatch
+        self.config = Config()
+        self.config.graceful_timeout = graceful(options) + ABORT_GRACE_PERIOD
+        for name, value in options.get("config", {}).items():
+            setattr(self.config, name, value)
+        self.started = False
+        self._exit: asyncio.Event | None = None
+        self._loop: asyncio.AbstractEventLoop | None = None
+        self._should_exit = False
+
+    @property
+    def should_exit(self) -> bool:
+        return self._should_exit
+
+    @should_exit.setter
+    def should_exit(self, value: bool) -> None:
+        self._should_exit = value
+        if value and self._loop is not None and self._exit is not None:
+            self._loop.call_soon_threadsafe(self._exit.set)
+
+    async def serve(self, sockets: list[socket.socket]) -> None:
+        self._loop = asyncio.get_running_loop()
+        self._exit = asyncio.Event()
+        if self._should_exit:
+            return
+        secure = self.config.ssl_enabled
+        bound = Sockets(
+            secure_sockets=sockets if secure else [],
+            insecure_sockets=[] if secure else sockets,
+            quic_sockets=[],
+        )
+        self.started = True
+        await worker_serve(
+            ASGIWrapper(cast("Any", self.dispatch)),
+            self.config,
+            sockets=bound,
+            shutdown_trigger=self._exit.wait,
+        )
+
+
+def make(**options: Unpack[Options]) -> Effect[HttpServerService, SocketServerError, ScopeTag]:
+    """An ``HttpServer`` on hypercorn, listening on ``host:port`` until the
+    scope closes."""
+    return make_listener(options, lambda dispatch: _Hypercorn(dispatch, options))
+
+
+def layer_server(**options: Unpack[Options]) -> Layer[HttpServer, SocketServerError]:
+    return layer_effect(HttpServer, make(**options))
+
+
+def layer(
+    **options: Unpack[Options],
+) -> Layer[HttpServer | Path | HttpPlatform | FileSystem | Generator, SocketServerError]:
+    """The server and the services its apps use (``live_server.layer_services``)."""
+    return merge(layer_server(**options), layer_services)
+
+
+def layer_test(
+    **options: Unpack[Options],
+) -> Layer[
+    HttpServer | Path | HttpPlatform | FileSystem | Generator | HttpClient, SocketServerError
+]:
+    """A server on a free loopback port, its services, and an ``HttpClient``
+    (httpx) sending to it."""
+    server = layer(**{"host": "127.0.0.1", **options})
+    return layer_test_client.pipe(provide_merge(httpx_layer)).pipe(provide_merge(server))
+
+
+__all__ = ["Options", "SocketServerError", "layer", "layer_server", "layer_test", "make"]

pyfect/http/incoming_message.py

@@ -0,0 +1,140 @@
+"""What incoming messages (responses here) share, and the schema decoders
+of their bodies and headers (Effect's ``HttpIncomingMessage``)."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Protocol, Unpack
+
+from pyfect._json_text import Json
+from pyfect.byte_size import ByteSize, mebibytes
+from pyfect.context import Reference
+from pyfect.effect import Effect
+from pyfect.http.headers import Headers
+from pyfect.http.url_params import UrlParams
+from pyfect.schema import (
+    Codec,
+    SchemaError,
+    Unknown,
+    decode_effect,
+    decode_unknown_effect,
+    from_json_string,
+    to_codec_json,
+)
+from pyfect.schema._options import ParseOptions
+from pyfect.stream import Stream
+
+type Reviver = Callable[[str, object], object]
+"""``JSON.parse``'s reviver: called with each key (``""`` for the root) and
+its parsed value, children first; what it returns replaces the value."""
+
+
+class HttpIncomingMessage[E](Protocol):
+    @property
+    def headers(self) -> Headers: ...
+    @property
+    def remote_address(self) -> str | None: ...
+    @property
+    def json(self) -> Effect[Json, E]: ...
+    @property
+    def text(self) -> Effect[str, E]: ...
+    @property
+    def url_params_body(self) -> Effect[UrlParams, E]: ...
+    @property
+    def bytes_(self) -> Effect[bytes, E]: ...
+    @property
+    def stream(self) -> Stream[bytes, E]: ...
+
+
+class MaxBodySize(Reference[ByteSize | None]):
+    """The most bytes a body read takes, 10 MiB by default (``None``: no
+    limit); a larger body fails as ``BodyTooLarge``. Effect has no default
+    limit; a request body is the client's to size, so pyfect bounds it."""
+
+    @classmethod
+    def default(cls) -> ByteSize | None:
+        return mebibytes(10)
+
+
+class MessageDecoder[T, RD](Protocol):
+    """Decodes a message, failing with the message's error or a
+    ``SchemaError``."""
+
+    def __call__[E](self, message: HttpIncomingMessage[E], /) -> Effect[T, E | SchemaError, RD]: ...
+
+
+def revive(value: object, reviver: Reviver) -> object:
+    """``value`` (parsed JSON) walked as ``JSON.parse`` walks it with
+    ``reviver``."""
+
+    def walk(key: str, node: object) -> object:
+        if isinstance(node, list):
+            items: list[object] = node  # pyright: ignore[reportUnknownVariableType]
+            node = [walk(str(i), item) for i, item in enumerate(items)]
+        elif isinstance(node, dict):
+            entries: dict[str, object] = node  # pyright: ignore[reportUnknownVariableType]
+            node = {k: walk(k, v) for k, v in entries.items()}
+        return reviver(key, node)
+
+    return walk("", value)
+
+
+_parse_json = decode_effect(from_json_string(Unknown))
+
+
+def schema_body_json[T, RD](
+    schema: Codec[T, object, RD, object],
+    *,
+    reviver: Reviver | None = None,
+    **options: Unpack[ParseOptions],
+) -> MessageDecoder[T, RD]:
+    """A decoder of a message's JSON body by ``schema``. With a ``reviver``
+    the text is parsed with it; an empty body is ``None`` either way."""
+    decode = decode_unknown_effect(to_codec_json(schema), **options)
+
+    def run[E](message: HttpIncomingMessage[E]) -> Effect[T, E | SchemaError, RD]:
+        if reviver is None:
+            return message.json.flat_map(decode)
+        revive_with = reviver
+
+        def from_text(body: str) -> Effect[T, E | SchemaError, RD]:
+            if body == "":
+                return message.json.flat_map(decode)
+            return _parse_json(body).flat_map(lambda parsed: decode(revive(parsed, revive_with)))
+
+        return message.text.flat_map(from_text)
+
+    return run
+
+
+def schema_body_url_params[T, RD](
+    schema: Codec[T, object, RD, object], **options: Unpack[ParseOptions]
+) -> MessageDecoder[T, RD]:
+    """A decoder of a form-encoded body: ``schema`` gets each name mapped
+    to its value, or to its values when repeated."""
+    decode = decode_unknown_effect(schema, **options)
+
+    def run[E](message: HttpIncomingMessage[E]) -> Effect[T, E | SchemaError, RD]:
+        return message.url_params_body.flat_map(lambda params: decode(params.to_record()))
+
+    return run
+
+
+def schema_headers[T, RD](
+    schema: Codec[T, object, RD, object], **options: Unpack[ParseOptions]
+) -> Callable[[HttpIncomingMessage[object]], Effect[T, SchemaError, RD]]:
+    """A decoder of a message's headers (lowercased names) by ``schema``."""
+    decode = decode_unknown_effect(schema, **options)
+    return lambda message: decode(dict(message.headers))
+
+
+__all__ = [
+    "HttpIncomingMessage",
+    "MaxBodySize",
+    "MessageDecoder",
+    "Reviver",
+    "revive",
+    "schema_body_json",
+    "schema_body_url_params",
+    "schema_headers",
+]