pyfect-dev 0.1.0__py3-none-any.whl

pyfect/http/__init__.py

@@ -0,0 +1,125 @@
+"""HTTP (Effect's ``effect/http``).
+
+Client:
+
+- ``client``: the ``HttpClient`` service, its combinators, ``make`` and
+  ``layer_test``
+- ``httpx_client``: an ``HttpClient`` on httpx (``pyfect-dev[httpx]``)
+- ``request`` / ``response``: requests and responses
+- ``body``, ``headers``, ``url_params``, ``url``, ``cookies``, ``method``:
+  the values they are made of
+- ``error``: ``HttpClientError`` and its reasons
+- ``trace_context``: ``traceparent`` / ``b3`` headers
+
+Server:
+
+- ``server``: the ``HttpServer`` service, ``serve`` and the test client;
+  ``live_server``: the stdlib HTTP/1.1 server behind it;
+  ``uvicorn_server``: one on uvicorn (``pyfect-dev[server]``);
+  ``hypercorn_server``: one that also speaks HTTP/2 (``pyfect-dev[http2]``)
+- ``asgi``: apps as ASGI applications (``to_asgi``), and ASGI applications
+  (FastAPI, Starlette) as apps (``from_asgi``)
+- ``server_request`` / ``server_response`` / ``server_error`` /
+  ``server_respondable``: what a handler reads, returns and fails with
+- ``router`` (on ``find_my_way``), ``middleware``, ``http_effect``,
+  ``web_handler``: building and running apps
+- ``origin``: which browser origins may open WebSockets and call RPCs
+  (same-origin by default)
+- ``platform``, ``static_server``, ``multipart``, ``etag``, ``mime``,
+  ``template``, ``status``
+- ``prometheus``: the metrics registry as a scrape endpoint
+
+::
+
+    from pyfect.http import client, httpx_client, response
+
+    @fx.fn
+    def program():
+        res = yield from client.get("https://example.com")
+        return (yield from res.text)
+
+    fx.run_blocking(program().pipe(fx.provide(httpx_client.layer)))
+"""
+
+from pyfect.http import (
+    abort,
+    asgi,
+    body,
+    client,
+    cookies,
+    error,
+    etag,
+    find_my_way,
+    headers,
+    http_effect,
+    incoming_message,
+    live_server,
+    method,
+    middleware,
+    mime,
+    multipart,
+    origin,
+    platform,
+    prometheus,
+    request,
+    response,
+    router,
+    server,
+    server_error,
+    server_request,
+    server_respondable,
+    server_response,
+    static_server,
+    status,
+    template,
+    trace_context,
+    url,
+    url_params,
+    web_handler,
+)
+from pyfect.http.client import HttpClient, HttpClientService
+from pyfect.http.error import HttpClientError
+from pyfect.http.request import HttpClientRequest
+from pyfect.http.response import HttpClientResponse
+
+__all__ = [
+    "HttpClient",
+    "HttpClientError",
+    "HttpClientRequest",
+    "HttpClientResponse",
+    "HttpClientService",
+    "abort",
+    "asgi",
+    "body",
+    "client",
+    "cookies",
+    "error",
+    "etag",
+    "find_my_way",
+    "headers",
+    "http_effect",
+    "incoming_message",
+    "live_server",
+    "method",
+    "middleware",
+    "mime",
+    "multipart",
+    "origin",
+    "platform",
+    "prometheus",
+    "request",
+    "response",
+    "router",
+    "server",
+    "server_error",
+    "server_request",
+    "server_respondable",
+    "server_response",
+    "static_server",
+    "status",
+    "template",
+    "trace_context",
+    "url",
+    "url_params",
+    "web_handler",
+]

pyfect/http/_asgi_listener.py

@@ -0,0 +1,272 @@
+"""What ``uvicorn_server`` and ``hypercorn_server`` share: a socket bound when
+the server is made, an ASGI engine on the program's loop (or a loop thread
+shared by such servers), and the shutdown as ``NodeHttpServer``'s.
+"""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+import asyncio
+import socket
+import threading
+from collections.abc import Callable, Coroutine
+from concurrent.futures import Future
+from typing import Any, Never, NotRequired, Protocol, TypedDict
+
+from pyfect._core import Effect, _WithFiber
+from pyfect._fiber import Fiber
+from pyfect._loop_thread import LoopThread
+from pyfect._os_wait import Resume, WaitsFor, os_wait
+from pyfect._root import run_loop
+from pyfect.context import Context
+from pyfect.duration import Duration, Seconds, to_seconds
+from pyfect.effect import fail, succeed, suspend, sync
+from pyfect.exit import Exit
+from pyfect.http.asgi._serve import AsgiHandler
+from pyfect.http.asgi._types import Receive, Scope, Send
+from pyfect.http.server import HttpApp, HttpServerService, Middleware
+from pyfect.http.server import make as make_server
+from pyfect.net.net_address import inet_address_from_native_unsafe
+from pyfect.run import run_fork
+from pyfect.scope import CloseableScope, acquire_release
+from pyfect.scope import Scope as ScopeTag
+from pyfect.socket.server import SocketServerError, SocketServerOpenError
+
+
+class ListenOptions(TypedDict):
+    port: NotRequired[int]
+    """0 (the default): any free port."""
+    host: NotRequired[str]
+    """Every interface by default (IPv6 and IPv4 where the host has both)."""
+    graceful_shutdown_timeout: NotRequired[Duration]
+    """How long requests in flight may run once the server closes (20s)."""
+    disable_preemptive_shutdown: NotRequired[bool]
+    """Give requests in flight no time: closing interrupts them at once."""
+
+
+class Engine(Protocol):
+    """An ASGI server that runs until ``should_exit`` is set."""
+
+    @property
+    def started(self) -> bool: ...
+
+    @property
+    def should_exit(self) -> bool: ...
+
+    @should_exit.setter
+    def should_exit(self, value: bool) -> None: ...
+
+    def serve(self, sockets: list[socket.socket]) -> Coroutine[Any, Any, None]: ...
+
+
+def _bind(host: str | None, port: int) -> socket.socket:
+    if host is None:
+        if socket.has_dualstack_ipv6():
+            return socket.create_server(("::", port), family=socket.AF_INET6, dualstack_ipv6=True)
+        return socket.create_server(("0.0.0.0", port))
+    family, _, _, _, address = socket.getaddrinfo(
+        host, port, type=socket.SOCK_STREAM, flags=socket.AI_PASSIVE
+    )[0]
+    return socket.create_server(address[:2], family=family)
+
+
+class Dispatch:
+    """The ASGI app the engine runs: the served app's handler, or a 503."""
+
+    __slots__ = ("current",)
+
+    def __init__(self) -> None:
+        self.current: AsgiHandler | None = None
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        handler = self.current
+        if handler is not None:
+            await handler(scope, receive, send)
+        elif scope["type"] == "http":
+            await send({"type": "http.response.start", "status": 503, "headers": []})
+            await send({"type": "http.response.body", "body": b""})
+        elif scope["type"] == "websocket":
+            await send({"type": "websocket.close", "code": 1013})
+
+
+class _Shared:
+    lock = threading.Lock()
+    loop: asyncio.AbstractEventLoop | None = None
+
+
+def _loop_thread() -> asyncio.AbstractEventLoop:
+    """The loop servers outside an asyncio program run on: one per process,
+    on a daemon thread, started on first use and never stopped. Fibers run
+    on it, and a fiber forked from a request can outlive its server (held
+    by a layer released later), so stopping the loop with the server would
+    strand it: interrupting it would wait for ever (DOGFOOD pulse #5)."""
+    with _Shared.lock:
+        if _Shared.loop is None:
+            _Shared.loop = LoopThread("pyfect-asgi").loop
+        return _Shared.loop
+
+
+def _wait[A](future: Future[A], waits_for: WaitsFor) -> Effect[A, BaseException]:
+    """Wait for ``future`` (done on any thread) without blocking."""
+
+    def register(resume: Resume[A, BaseException]) -> None:
+        def done(finished: Future[A]) -> None:
+            if finished.cancelled():
+                resume(fail(asyncio.CancelledError()))
+                return
+            error = finished.exception()
+            resume(succeed(finished.result()) if error is None else fail(error))
+
+        future.add_done_callback(done)
+
+    return os_wait(register, waits_for=waits_for)
+
+
+class Running:
+    """The engine serving on ``loop`` until ``stop``."""
+
+    def __init__(
+        self,
+        server: Engine,
+        loop: asyncio.AbstractEventLoop,
+        graceful: Seconds,
+    ) -> None:
+        self.server = server
+        self.graceful = graceful
+        self.loop = loop
+        self.served: Future[None] | None = None
+
+    def submit[A](self, make: Callable[[], Coroutine[Any, Any, A]]) -> Future[A]:
+        return asyncio.run_coroutine_threadsafe(make(), self.loop)
+
+    def start(self, sock: socket.socket) -> Effect[None, BaseException]:
+        self.served = self.submit(lambda: self.server.serve(sockets=[sock]))
+        served = self.served
+
+        async def started() -> None:
+            while not self.server.started and not served.done():
+                await asyncio.sleep(0.005)
+            if served.done():
+                served.result()
+                raise RuntimeError("the server stopped while starting")
+
+        return _wait(self.submit(started), "work")
+
+    def stop(self) -> Effect[None]:
+        """Stop accepting and wait for the engine to exit (its own
+        graceful timeout cancels what is still running)."""
+
+        def body() -> Effect[None]:
+            self.server.should_exit = True
+            assert self.served is not None
+            return _wait(self.served, "event").ignore()
+
+        return suspend(body)
+
+    def shut_down(
+        self, handler: AsgiHandler, requests: CloseableScope, exit: Exit[Any, Any]
+    ) -> Effect[None]:
+        """As ``NodeHttpServer``'s preemptive shutdown: stop accepting and
+        give requests in flight ``graceful`` seconds, then interrupt the
+        request fibers (``requests``) and end the bodies still streaming, so
+        the engine exits before its own timeout would cancel them."""
+
+        def end_responses() -> Effect[None]:
+            async def end() -> None:
+                handler.end_responses()
+
+            return _wait(self.submit(end), "work").ignore()
+
+        def body() -> Effect[None]:
+            self.server.should_exit = True
+            assert self.served is not None
+            served = _wait(self.served, "event").ignore()
+            return (
+                served.timeout_option(self.graceful)
+                .and_then(requests.close(exit))
+                .and_then(suspend(end_responses))
+                .and_then(self.stop())
+            )
+
+        return suspend(body)
+
+
+def graceful(options: ListenOptions) -> Seconds:
+    if options.get("disable_preemptive_shutdown", False):
+        return to_seconds(0)
+    return to_seconds(options.get("graceful_shutdown_timeout", 20))
+
+
+def _serve(
+    dispatch: Dispatch, running: Running
+) -> Callable[[HttpApp, Middleware | None], Effect[None, Never, ScopeTag]]:
+    def serve(app: HttpApp, middleware: Middleware | None) -> Effect[None, Never, ScopeTag]:
+        def body(fiber: Fiber[Any, Any]) -> Effect[None, Never, ScopeTag]:
+            context: Context[Any] = fiber._context
+            serve_scope = context.get_unsafe(ScopeTag)
+            requests: CloseableScope = serve_scope.fork_unsafe("parallel")
+
+            def fork(
+                handled: Effect[None, Never, Any], request: Context[Any]
+            ) -> Fiber[None, Never]:
+                return run_fork(handled, context=request).run_in(requests)
+
+            handler = AsgiHandler(app, middleware, context, fork)
+            dispatch.current = handler
+
+            def unserve(exit: Exit[Any, Any]) -> Effect[None]:
+                def remove() -> None:
+                    if dispatch.current is handler:
+                        dispatch.current = None
+
+                return running.shut_down(handler, requests, exit).ensuring(sync(remove))
+
+            return serve_scope.add_finalizer(unserve)
+
+        return _WithFiber(body)
+
+    return serve
+
+
+def make_listener(
+    options: ListenOptions, engine: Callable[[Dispatch], Engine]
+) -> Effect[HttpServerService, SocketServerError, ScopeTag]:
+    """An ``HttpServer`` on ``engine``, listening until the scope closes."""
+
+    def bind() -> Effect[socket.socket, SocketServerError]:
+        try:
+            return succeed(_bind(options.get("host"), options.get("port", 0)))
+        except OSError as cause:
+            return fail(SocketServerError(SocketServerOpenError(cause)))
+
+    def run(sock: socket.socket) -> Effect[HttpServerService, SocketServerError, ScopeTag]:
+        dispatch = Dispatch()
+
+        def start() -> Effect[Running, SocketServerError]:
+            current = run_loop()
+            loop = current or _loop_thread()
+            running = Running(engine(dispatch), loop, graceful=graceful(options))
+            return (
+                running.start(sock)
+                .as_(running)
+                .catch_all(
+                    lambda cause: running.stop().and_then(
+                        fail(SocketServerError(SocketServerOpenError(cause)))
+                    )
+                )
+            )
+
+        def service(running: Running) -> HttpServerService:
+            host, port = sock.getsockname()[:2]
+            address = inet_address_from_native_unsafe(host, port)
+            return make_server(serve=_serve(dispatch, running), address=address)
+
+        return acquire_release(suspend(start), lambda running, _: running.stop()).map(service)
+
+    def close(sock: socket.socket, _exit: object) -> Effect[None]:
+        return sync(sock.close)
+
+    return acquire_release(suspend(bind), close).flat_map(run)
+
+
+__all__ = ["Dispatch", "Engine", "ListenOptions", "Running", "graceful", "make_listener"]

pyfect/http/_authority.py

@@ -0,0 +1,63 @@
+"""The authority of a request URL (``host[:port]``), checked once where a
+server reads it from the client (``Host``, an absolute-form target,
+``X-Forwarded-Host``). A request's absolute URL is built only from an
+``Authority`` and an ``OriginForm``, so nothing a client sends as a host
+or a target can become part of the other."""
+
+from __future__ import annotations
+
+import re
+from ipaddress import IPv6Address
+from typing import NewType
+
+Authority = NewType("Authority", str)
+"""A ``host[:port]``: a bracketed IP literal or a name of letters, digits,
+``.``, ``-`` and ``_``, and a port up to 65535."""
+
+_SHAPE = re.compile(r"(?:\[([0-9A-Fa-f:.]+)\]|[A-Za-z0-9._-]+)(?::([0-9]{1,5}))?")
+
+
+def parse_authority(text: str) -> Authority | None:
+    """``text`` as an authority; ``None`` for anything else (a path, query
+    or fragment, userinfo, a list, a bad IP literal or port)."""
+    match = _SHAPE.fullmatch(text)
+    if match is None or (match[2] is not None and int(match[2]) > 65535):
+        return None
+    if match[1] is not None:
+        try:
+            IPv6Address(match[1])
+        except ValueError:
+            return None
+    return Authority(text)
+
+
+LOCALHOST = Authority("localhost")
+"""The authority of a request that names none."""
+
+OriginForm = NewType("OriginForm", str)
+"""A path and query that starts with ``/`` (RFC 9112's origin-form)."""
+
+ROOT = OriginForm("/")
+
+
+def parse_origin_form(target: str) -> OriginForm | None:
+    """``target`` as a path and query; ``None`` without a leading ``/``
+    (``admin``, ``evil.test/admin``), which would run into the authority
+    or route somewhere a front end did not see."""
+    return OriginForm(target) if target.startswith("/") else None
+
+
+def absolute_url(scheme: str, authority: Authority, target: OriginForm) -> str:
+    """The URL of ``target`` at ``authority``."""
+    return f"{scheme}://{authority}{target}"
+
+
+__all__ = [
+    "LOCALHOST",
+    "ROOT",
+    "Authority",
+    "OriginForm",
+    "absolute_url",
+    "parse_authority",
+    "parse_origin_form",
+]

pyfect/http/_brotli.py

@@ -0,0 +1,27 @@
+"""Brotli through the ``brotli`` extra; ``http.platform`` offers ``br`` when it is installed."""
+
+from typing import Any
+
+from pyfect._extras import missing_extra
+
+try:
+    import brotli as _library  # pyright: ignore[reportMissingTypeStubs]
+except ImportError as error:
+    raise missing_extra("Brotli compression", "brotli") from error
+
+_NODE_DEFAULT_QUALITY = 11
+
+
+class Brotli:
+    __slots__ = ("_compressor",)
+
+    def __init__(self, level: int | None) -> None:
+        quality = _NODE_DEFAULT_QUALITY if level is None else level
+        library: Any = _library
+        self._compressor: Any = library.Compressor(quality=quality)
+
+    def chunk(self, data: bytes) -> bytes:
+        return self._compressor.process(data) + self._compressor.flush()
+
+    def finish(self) -> bytes:
+        return self._compressor.finish()

pyfect/http/_compression.py

@@ -0,0 +1,52 @@
+"""Compressing request bodies (Effect's ``internal/compressionBody``)."""
+
+from __future__ import annotations
+
+import zlib
+from typing import Literal
+
+from pyfect.effect import Effect, succeed, sync
+from pyfect.http import body as _body
+from pyfect.stream import Stream
+from pyfect.stream import suspend as suspend_stream
+from pyfect.stream import sync as sync_stream
+
+type CompressionAlgorithm = Literal["gzip", "deflate"]
+
+_WBITS: dict[CompressionAlgorithm, int] = {"gzip": 31, "deflate": 15}
+
+
+def _compressor(algorithm: CompressionAlgorithm) -> zlib._Compress:  # pyright: ignore[reportPrivateUsage]
+    return zlib.compressobj(wbits=_WBITS[algorithm])
+
+
+def _compress_stream(
+    chunks: Stream[bytes, object], algorithm: CompressionAlgorithm
+) -> Stream[bytes, object]:
+    def make() -> Stream[bytes, object]:
+        compressor = _compressor(algorithm)
+        compressed = chunks.map(compressor.compress).concat(sync_stream(compressor.flush))
+        return compressed.filter(lambda chunk: len(chunk) > 0)
+
+    return suspend_stream(make)
+
+
+def compress_body(
+    body: _body.HttpBody, algorithm: CompressionAlgorithm, min_size: int = 1024
+) -> Effect[_body.HttpBody]:
+    """``body`` compressed with ``algorithm``; bodies other than bytes and
+    streams, and those known to be shorter than ``min_size``, unchanged. A
+    compressed stream has no known length."""
+    if not isinstance(body, _body.Bytes | _body.Stream):
+        return succeed(body)
+    if body.content_length is not None and body.content_length < min_size:
+        return succeed(body)
+    if isinstance(body, _body.Bytes):
+        data, content_type = body.body, body.content_type
+
+        def compress() -> _body.HttpBody:
+            compressor = _compressor(algorithm)
+            return _body.bytes_(compressor.compress(data) + compressor.flush(), content_type)
+
+        return sync(compress)
+    return succeed(_body.stream(_compress_stream(body.stream, algorithm), body.content_type))

pyfect/http/_field.py

@@ -0,0 +1,18 @@
+"""RFC 9110 field names and values, as every server sends them: a name is
+a token, a value is tab, visible ASCII and obs-text (no other control
+character)."""
+
+from __future__ import annotations
+
+import re
+
+_TOKEN = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
+_VALUE = re.compile(r"[\t\x20-\x7e\x80-\xff]*")
+
+
+def is_token(name: str) -> bool:
+    return _TOKEN.fullmatch(name) is not None
+
+
+def is_field_value(value: str) -> bool:
+    return _VALUE.fullmatch(value) is not None

pyfect/http/_foreign_path.py

@@ -0,0 +1,27 @@
+"""The path a foreign handler (an ASGI app, a Web handler) is given must
+split into the segments the router matched: ``%2F`` stays encoded, and a
+dot segment, which the router matched literally but the handler may
+resolve, is refused."""
+
+from __future__ import annotations
+
+import re
+from urllib.parse import unquote
+
+_ENCODED_SLASH = re.compile("(%2[fF])")
+
+
+def decode_path(path: str) -> str:
+    """``path`` percent-decoded but for ``%2F``."""
+    pieces = _ENCODED_SLASH.split(path)
+    return "".join(piece if i % 2 else unquote(piece) for i, piece in enumerate(pieces))
+
+
+def has_dot_segment(path: str) -> bool:
+    """Whether a segment of ``path`` is ``.`` or ``..`` once decoded, with
+    ``\\`` as a separator too (WHATWG URLs and Windows paths split there)."""
+    segments = decode_path(path).replace("\\", "/").split("/")
+    return any(segment in (".", "..") for segment in segments)
+
+
+__all__ = ["decode_path", "has_dot_segment"]

pyfect/http/_form_data.py

@@ -0,0 +1,34 @@
+"""``multipart/form-data`` encoding of a ``FormData`` body, as ``fetch``
+writes one."""
+
+from __future__ import annotations
+
+import secrets
+
+from pyfect._js import text_encode
+from pyfect.http import body as _body
+
+
+def _quote(text: str) -> str:
+    return text.replace("\r", "%0D").replace("\n", "%0A").replace('"', "%22")
+
+
+def encode(form: _body.FormData) -> tuple[bytes, str]:
+    """The encoded form and its content type (with the boundary)."""
+    boundary = f"----formdata-pyfect-{secrets.token_hex(12)}"
+    out = bytearray()
+    for name, value in form.entries:
+        out += text_encode(f"--{boundary}\r\n")
+        if isinstance(value, str):
+            out += text_encode(f'Content-Disposition: form-data; name="{_quote(name)}"\r\n\r\n')
+            out += text_encode(value) + b"\r\n"
+        else:
+            disposition = (
+                f'Content-Disposition: form-data; name="{_quote(name)}"; '
+                f'filename="{_quote(value.name)}"\r\n'
+            )
+            out += text_encode(disposition)
+            out += text_encode(f"Content-Type: {value.content_type}\r\n\r\n")
+            out += value.content + b"\r\n"
+    out += text_encode(f"--{boundary}--\r\n")
+    return bytes(out), f"multipart/form-data; boundary={boundary}"