pyfect-dev 0.1.0__py3-none-any.whl

pyfect/http/live_server/_serve.py

@@ -0,0 +1,225 @@
+"""Serving one connection: read a head, hand the request to the app, write
+its response, and keep the connection for the next request unless it must
+close (a ``connection: close``, a body too long to drain, a broken
+response, a shutdown)."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from http import HTTPStatus
+from typing import Any
+
+from pyfect._core import Effect
+from pyfect._fiber import Fiber
+from pyfect.duration import Seconds
+from pyfect.effect import gen, succeed, sync, void
+from pyfect.http._authority import LOCALHOST, absolute_url
+from pyfect.http.headers import Headers
+from pyfect.http.live_server._exchange import MAX_DRAIN_BYTES, Exchange
+from pyfect.http.live_server._head import HeadError, RequestHead, parse_head
+from pyfect.http.live_server._inbound import BareLineFeed, HeadTooLarge, Inbound
+from pyfect.http.live_server._listener import ConnectionState, Listener
+from pyfect.http.live_server._timers import timeout_option
+from pyfect.http.live_server._upgrade import upgrade
+from pyfect.http.server_error import CLIENT_ABORT_ANNOTATION
+from pyfect.http.server_request import RequestProtocol, SourceRequest
+from pyfect.option import Option
+from pyfect.socket._connection import Connection
+from pyfect.socket._errors import SocketError
+
+MAX_HEAD_SIZE = 16 * 1024
+"""Node's ``maxHeaderSize``: a longer head is answered with 431."""
+
+CLOSE_GRACE_PERIOD = Seconds(0.5)
+"""How long a closing connection waits for the client to stop sending
+before it is torn down (so the response is not lost to a reset)."""
+
+type Handle = Callable[[SourceRequest, Exchange], Effect[None]]
+
+
+@dataclass(frozen=True, slots=True)
+class Timeouts:
+    """Node's ``keepAliveTimeout`` (an idle connection between requests is
+    closed), ``headersTimeout`` (a head must arrive within it once it is
+    awaited, or started after a keep-alive wait; a started one is answered
+    with 408) and ``requestTimeout`` (the body must have arrived within it
+    of the head arriving, or its read fails, the response drain
+    included, and the connection closes)."""
+
+    keep_alive: Seconds
+    headers: Seconds
+    request: Seconds
+
+
+def _reject(conn: Connection, status: int) -> Effect[None]:
+    head = (
+        f"HTTP/1.1 {status} {HTTPStatus(status).phrase}\r\n"
+        "connection: close\r\ncontent-length: 0\r\n\r\n"
+    ).encode("ascii")
+
+    def ignore(_: SocketError) -> Effect[None]:
+        return void
+
+    return conn.write((head,)).catch_all(ignore).flat_map(lambda _: _close(conn))
+
+
+def _close(conn: Connection) -> Effect[None]:
+    """End the response side, read on for the grace period, then close."""
+
+    def start() -> None:
+        conn.end_writing()
+        conn.discard_input()
+
+    waited = timeout_option(conn.closed(), CLOSE_GRACE_PERIOD)
+    return (
+        sync(start)
+        .flat_map(lambda _: waited)
+        .flat_map(lambda _: sync(lambda: conn.close(graceful=False)))
+    )
+
+
+def _peer_address(conn: Connection) -> str | None:
+    name: Any = conn.peer_name()
+    if isinstance(name, tuple) and name:
+        return str(name[0])  # pyright: ignore[reportUnknownArgumentType]
+    return None
+
+
+def _request(
+    exchange: Exchange,
+    protocol: RequestProtocol,
+    remote_address: str | None,
+    websocket_max_message_size: int,
+) -> SourceRequest:
+    head = exchange.head
+    url = head.target
+    body = exchange.body
+    request: SourceRequest | None = None
+
+    def upgrade_request(upgrade_headers: Headers) -> Effect[Any, Any]:
+        assert request is not None
+        return upgrade(exchange, request, upgrade_headers, websocket_max_message_size)
+
+    request = SourceRequest(
+        head.method,
+        url if url == "*" else absolute_url(protocol, head.authority or LOCALHOST, url),
+        head.headers,
+        None if body.complete else body.stream,
+        source=exchange,
+        remote_address=remote_address,
+        protocol=protocol,
+        upgrade=upgrade_request if head.upgrade is not None else None,
+    )
+    return request
+
+
+def _run_request(exchange: Exchange, handled: Effect[None]) -> Effect[None]:
+    """Run the request on a fiber interrupted as a client abort if the
+    client goes first: once it stops sending (Node's servers do not allow
+    half-open connections), or for an upgrade once the connection is
+    lost."""
+    conn = exchange.conn
+    gone = conn.closed() if exchange.head.upgrade is not None else conn.read_ended()
+
+    def body():
+        fiber: Fiber[None, Any] = yield from handled.fork()
+
+        def abort(_: None) -> Effect[None]:
+            if exchange.finished:
+                return void
+            exchange.close_after = True
+            exchange.aborted = True
+            return sync(lambda: fiber.interrupt_as_unsafe(None, CLIENT_ABORT_ANNOTATION))
+
+        watcher = yield from gone.flat_map(abort).fork()
+        yield from fiber.await_exit()
+        yield from watcher.interrupt()
+
+    return gen(body)
+
+
+def serve_connection(
+    listener: Listener,
+    state: ConnectionState,
+    handle: Handle,
+    protocol: RequestProtocol,
+    timeouts: Timeouts,
+    *,
+    websocket_max_message_size: int,
+) -> Effect[None]:
+    conn = state.conn
+    inbound = Inbound(conn)
+    remote_address = _peer_address(conn)
+
+    def head_failed(
+        error: SocketError | HeadTooLarge | BareLineFeed,
+    ) -> Effect[bytes | int | None]:
+        if isinstance(error, HeadTooLarge):
+            return succeed(431)
+        return succeed(400 if isinstance(error, BareLineFeed) else None)
+
+    def within_headers_timeout(found: Option[bytes | int | None]) -> bytes | int | None:
+        if found is None:
+            return 408 if inbound.has_pending else None
+        return found.value
+
+    def next_head(first: bool) -> Effect[bytes | int | None]:
+        """The next head, or the status to reject it with; ``None`` to
+        close quietly."""
+        head = timeout_option(
+            inbound.read_head(MAX_HEAD_SIZE).catch_all(head_failed), timeouts.headers
+        ).map(within_headers_timeout)
+        if first:
+            return head
+
+        def started(_: SocketError) -> Effect[None]:
+            return void  # the head read sees the failure
+
+        waited = timeout_option(inbound.wait_data().catch_all(started), timeouts.keep_alive)
+        return waited.flat_map(lambda found: succeed(None) if found is None else head)
+
+    def parsed(data: bytes | int | None) -> RequestHead | int | None:
+        if not isinstance(data, bytes):
+            return data
+        try:
+            return parse_head(data)
+        except HeadError as error:
+            return error.status
+
+    def after(exchange: Exchange) -> Effect[bool]:
+        """Whether the connection serves another request."""
+        if exchange.upgraded is not None or not exchange.finished:
+            return succeed(False)
+        if exchange.close_after:
+            return _close(conn).as_(False)
+        return exchange.body.drain(MAX_DRAIN_BYTES).flat_map(
+            lambda drained: succeed(True) if drained else _close(conn).as_(False)
+        )
+
+    def loop():
+        first = True
+        # As Node's server, a connection busy when shutdown starts goes on
+        # serving keep-alive requests until it is idle for the keep-alive
+        # timeout (or the graceful shutdown timeout ends).
+        while True:
+            state.idle = True
+            inbound.deadline = None
+            data = yield from next_head(first)
+            inbound.deadline = time.monotonic() + timeouts.request
+            first = False
+            state.idle = False
+            head = parsed(data)
+            if head is None:
+                return
+            if isinstance(head, int):
+                yield from _reject(conn, head)
+                return
+            exchange = Exchange(conn, inbound, head, int(timeouts.keep_alive))
+            request = _request(exchange, protocol, remote_address, websocket_max_message_size)
+            yield from _run_request(exchange, handle(request, exchange))
+            if not (yield from after(exchange)):
+                return
+
+    return gen(loop).ensuring(sync(lambda: listener.done(state)))

pyfect/http/live_server/_timers.py

@@ -0,0 +1,20 @@
+"""The server's own timers (keep-alive, headers, close grace, shutdown)
+run on the system clock, as Node's socket timers do: a ``TestClock`` the
+program provides never moves them, so it must not stop them either."""
+
+from pyfect._core import Effect
+from pyfect.clock import Clock, LiveClock
+from pyfect.duration import Seconds
+from pyfect.effect import provide_service
+from pyfect.option import Option
+
+_LIVE = LiveClock()
+
+
+def timeout_option[A, E, R](eff: Effect[A, E, R], seconds: Seconds) -> Effect[Option[A], E, R]:
+    """``eff.timeout_option(seconds)`` timed by the system clock; ``eff``
+    is the server's own I/O, which reads no clock."""
+    return eff.timeout_option(seconds).pipe(provide_service(Clock, _LIVE))
+
+
+__all__ = ["timeout_option"]

pyfect/http/live_server/_upgrade.py

@@ -0,0 +1,108 @@
+"""Upgrading a connection to another protocol (RFC 9110 §7.8), as
+Effect's ``NodeHttpServer.makeUpgradeHandler``: ``Upgrade: websocket`` is a
+WebSocket (``_websocket``, which takes no custom response headers); for
+any other protocol the handler's ``request.upgrade`` answers ``101
+Switching Protocols`` once it takes the socket's reader, after the request
+body has ended (its unread rest discarded, up to ``MaxBodySize``), and the
+raw socket reads the bytes the client sent after the request first."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Never
+
+from pyfect._core import Effect
+from pyfect.effect import fail, succeed, suspend, sync, void
+from pyfect.http.headers import Headers
+from pyfect.http.incoming_message import MaxBodySize
+from pyfect.http.live_server._exchange import (
+    MAX_DRAIN_BYTES,
+    Exchange,
+    InvalidResponse,
+    encode_fields,
+)
+from pyfect.http.live_server._websocket import web_socket
+from pyfect.http.server_error import HttpServerError, InternalError, request_parse_error
+from pyfect.http.server_request import ServerRequest
+from pyfect.http.server_response import empty
+from pyfect.scope import Scope
+from pyfect.socket._errors import CloseEvent, SocketError, upgrade_error
+from pyfect.socket._socket import Chunk, Reader, SocketService, Writer, encode_chunk
+from pyfect.socket._socket import make as make_socket
+
+_RESERVED = frozenset({"connection", "upgrade", "content-length", "transfer-encoding"})
+_SWITCHED = empty(status=101)
+
+
+def _head(protocol: str, headers: Headers) -> bytes:
+    fields = [("Connection", "Upgrade"), ("Upgrade", protocol), *headers.items()]
+    return encode_fields("HTTP/1.1 101 Switching Protocols", fields)
+
+
+def _upgrade_error(
+    request: ServerRequest, description: str, cause: object = None
+) -> HttpServerError:
+    """Effect's ``upgradeError``: an ``InternalError``."""
+    return HttpServerError(InternalError(request, description, cause))
+
+
+def upgrade(
+    exchange: Exchange, request: ServerRequest, headers: Headers, websocket_max_message_size: int
+) -> Effect[SocketService, HttpServerError]:
+    """The connection of an ``Upgrade`` request: a WebSocket or raw."""
+    protocol = exchange.head.upgrade
+    if protocol is None:
+        return fail(request_parse_error(request, "Not an upgradeable ServerRequest"))
+    if protocol.lower() == "websocket":
+        if headers:
+            message = "WebSocket upgrades do not support custom response headers"
+            return fail(_upgrade_error(request, message))
+        return succeed(web_socket(exchange, websocket_max_message_size))
+    reserved = sorted(_RESERVED.intersection(headers))
+    if reserved:
+        return fail(_upgrade_error(request, f"Reserved upgrade header: {reserved[0]}"))
+    try:
+        head = _head(protocol, headers)
+    except InvalidResponse as cause:
+        return fail(_upgrade_error(request, str(cause), cause))
+    conn = exchange.conn
+
+    def switch() -> Effect[None, SocketError]:
+        if exchange.upgraded is not None:
+            return void
+        if exchange.headers_sent:
+            return fail(upgrade_error(RuntimeError("the response was already sent")))
+        exchange.headers_sent = True
+        exchange.upgraded = _SWITCHED
+        exchange.close_after = True
+        return conn.write((head,))
+
+    def drained(ok: bool) -> Effect[None, SocketError]:
+        if not ok:
+            return fail(upgrade_error(RuntimeError("the request body is too large")))
+        return suspend(switch)
+
+    def opened(_: None) -> Reader:
+        leftover = exchange.take_pending()
+        first: list[bytes] = [leftover] if leftover else []
+
+        def pull() -> Effect[tuple[bytes, ...], SocketError]:
+            return succeed((first.pop(),)) if first else conn.pull()
+
+        return Reader(suspend(pull))
+
+    def reader(limit: int | None) -> Effect[Reader, SocketError]:
+        maximum = MAX_DRAIN_BYTES if limit is None else limit
+        return exchange.body.drain(maximum).flat_map(drained).map(opened)
+
+    def write(chunk: Chunk | CloseEvent) -> Effect[None, SocketError]:
+        if isinstance(chunk, CloseEvent):
+            return sync(lambda: conn.close(graceful=chunk.code <= 1000))
+        return conn.write((encode_chunk(chunk),))
+
+    def write_all(chunks: Sequence[Chunk]) -> Effect[None, SocketError]:
+        return conn.write(tuple(encode_chunk(chunk) for chunk in chunks))
+
+    opened_reader: Effect[Reader, SocketError, Scope] = MaxBodySize().flat_map(reader)
+    writer: Effect[Writer, Never, Scope] = succeed(Writer(write, write_all))
+    return succeed(make_socket(opened_reader, writer))

pyfect/http/live_server/_websocket.py

@@ -0,0 +1,286 @@
+"""A WebSocket upgrade on the live server (Effect's ``NodeHttpServer``
+handing ``Upgrade: websocket`` requests to ``ws``'s ``handleUpgrade``).
+
+Acquiring the socket's reader checks the handshake as ``ws`` does (a bad
+one is answered with ``ws``'s 400 or 405 and fails the reader as a socket
+closed before the upgrade), sends the ``101`` with ``Sec-WebSocket-Accept``
+(and the client's first subprotocol), then speaks RFC 6455: pulls give
+the client's messages (text as ``str``), pings are answered, and the
+client's close is answered and fails the pull with its code (1006 when the
+connection ends without one). The reader's scope closes the WebSocket with
+the handler's code (1000, 1001 when interrupted, 1011 on failure) unless a
+close was already sent. Writes wait for the reader to open the socket; a
+``CloseEvent`` sends a close frame, and writes after a close are dropped,
+as ``ws`` drops them."""
+
+from __future__ import annotations
+
+import threading
+from collections import deque
+from collections.abc import Sequence
+from typing import Never
+
+from pyfect._core import Effect
+from pyfect._js import text_encode
+from pyfect.effect import fail, succeed, suspend, sync, void
+from pyfect.exit import Exit
+from pyfect.http._field import is_token
+from pyfect.http.live_server._exchange import Exchange, encode_fields
+from pyfect.http.live_server._ws_frames import (
+    Close,
+    FrameParser,
+    Message,
+    Opcode,
+    Ping,
+    ProtocolError,
+    accept_key,
+    encode_close,
+    encode_frame,
+    encode_message,
+    is_valid_close_code,
+    is_valid_key,
+)
+from pyfect.http.server_response import empty
+from pyfect.latch import Latch
+from pyfect.scope import Scope, acquire_release
+from pyfect.socket._errors import (
+    CloseEvent,
+    SocketError,
+    close_error,
+    open_error,
+    read_error,
+    write_error,
+)
+from pyfect.socket._socket import Chunk, Reader, SocketService, Writer, make
+from pyfect.socket._web_socket import close_code
+
+_SWITCHED = empty(status=101)
+
+
+class _Refused(Exception):
+    def __init__(self, status: int, phrase: str, message: str, *extra: tuple[str, str]) -> None:
+        super().__init__(message)
+        self.status = status
+        self.phrase = phrase
+        self.extra = extra
+
+    def response(self) -> bytes:
+        body = text_encode(str(self))
+        fields = [
+            ("Connection", "close"),
+            ("Content-Type", "text/html"),
+            ("Content-Length", str(len(body))),
+            *self.extra,
+        ]
+        return encode_fields(f"HTTP/1.1 {self.status} {self.phrase}", fields) + body
+
+
+def _subprotocols(value: str | None) -> list[str]:
+    if value is None:
+        return []
+    protocols = [part.strip(" \t") for part in value.split(",")]
+    if not all(map(is_token, protocols)) or any(
+        p in protocols[:i] for i, p in enumerate(protocols)
+    ):
+        raise _Refused(400, "Bad Request", "Invalid Sec-WebSocket-Protocol header")
+    return protocols
+
+
+def _switching_head(exchange: Exchange) -> bytes:
+    """The ``101``, or ``_Refused`` for a handshake ``ws`` refuses."""
+    head = exchange.head
+    headers = head.headers
+    if head.method != "GET":
+        raise _Refused(405, "Method Not Allowed", "Invalid HTTP method")
+    key = headers.get("sec-websocket-key")
+    if key is None or not is_valid_key(key):
+        raise _Refused(400, "Bad Request", "Missing or invalid Sec-WebSocket-Key header")
+    if headers.get("sec-websocket-version") not in ("13", "8"):
+        raise _Refused(
+            400,
+            "Bad Request",
+            "Missing or invalid Sec-WebSocket-Version header",
+            ("Sec-WebSocket-Version", "13, 8"),
+        )
+    protocols = _subprotocols(headers.get("sec-websocket-protocol"))
+    fields = [
+        ("Upgrade", "websocket"),
+        ("Connection", "Upgrade"),
+        ("Sec-WebSocket-Accept", accept_key(key)),
+    ]
+    if protocols:
+        fields.append(("Sec-WebSocket-Protocol", protocols[0]))
+    return encode_fields("HTTP/1.1 101 Switching Protocols", fields)
+
+
+def _closed_before_upgrade() -> SocketError:
+    return open_error(ConnectionError("The socket closed before the upgrade"))
+
+
+class _Session:
+    """One upgraded connection: what the reader parsed and has yet to hand
+    over or answer, and whether a close frame was sent."""
+
+    __slots__ = (
+        "_end_writing",
+        "_error",
+        "_leftover",
+        "_lock",
+        "_parser",
+        "_ready",
+        "_replies",
+        "close_sent",
+        "exchange",
+    )
+
+    def __init__(self, exchange: Exchange, max_message_size: int) -> None:
+        self.exchange = exchange
+        self._parser = FrameParser(max_message_size)
+        self._leftover = exchange.take_pending()
+        self._lock = threading.Lock()
+        self._ready: deque[Chunk] = deque()
+        self._replies: list[bytes] = []
+        self._end_writing = False
+        self._error: SocketError | None = None
+        self.close_sent = False
+
+    def claim_close(self) -> bool:
+        """Whether this call is the one that sends the close frame."""
+        with self._lock:
+            if self.close_sent:
+                return False
+            self.close_sent = True
+            return True
+
+    # --- reading ------------------------------------------------------------------
+
+    def _feed(self, chunks: Sequence[bytes]) -> None:
+        for chunk in chunks:
+            try:
+                events = self._parser.feed(chunk)
+            except ProtocolError as error:
+                self._error = read_error(error)
+                if self.claim_close():
+                    # ``ws`` closes with the code alone (``receiverOnError``).
+                    self._replies.append(encode_close(error.code))
+                self._end_writing = True
+                return
+            for event in events:
+                match event:
+                    case Message(data):
+                        self._ready.append(data)
+                    case Ping(payload):
+                        if not self.close_sent:
+                            self._replies.append(encode_frame(Opcode.PONG, payload))
+                    case Close(code, reason):
+                        self._error = close_error(code, reason)
+                        if self.claim_close():
+                            self._replies.append(
+                                encode_close(None if code == 1005 else code, reason)
+                            )
+                        self._end_writing = True
+
+    def _lost(self, _: SocketError) -> Effect[None]:
+        if self._error is None:
+            self._error = close_error(1006)
+        return void
+
+    def pull(self) -> Effect[tuple[Chunk, ...], SocketError]:
+        conn = self.exchange.conn
+
+        def step() -> Effect[tuple[Chunk, ...], SocketError]:
+            if self._replies:
+                replies = tuple(self._replies)
+                self._replies.clear()
+                return conn.write(replies).catch_all(self._lost).flat_map(lambda _: suspend(step))
+            if self._end_writing:
+                self._end_writing = False
+                conn.end_writing()
+            if self._ready:
+                ready = tuple(self._ready)
+                self._ready.clear()
+                return succeed(ready)
+            if self._error is not None:
+                return fail(self._error)
+            if self._leftover:
+                leftover, self._leftover = self._leftover, b""
+                self._feed((leftover,))
+                return suspend(step)
+            return (
+                conn.pull_into(self._feed).catch_all(self._lost).flat_map(lambda _: suspend(step))
+            )
+
+        return suspend(step)
+
+    # --- writing --------------------------------------------------------------------
+
+    def _send(self, frames: tuple[bytes, ...]) -> Effect[None, SocketError]:
+        # As ``ws``: a send on a closed connection is dropped, not raised;
+        # the reader sees the connection end.
+        return self.exchange.conn.write(frames).catch_all(lambda _: void)
+
+    def write(self, chunk: Chunk | CloseEvent) -> Effect[None, SocketError]:
+        if not isinstance(chunk, CloseEvent):
+            return void if self.close_sent else self._send((encode_message(chunk),))
+        reason = chunk.reason or ""
+        if not is_valid_close_code(chunk.code):
+            return fail(write_error(TypeError("First argument must be a valid error code number")))
+        if len(text_encode(reason)) > 123:
+            return fail(write_error(ValueError("The message must not be greater than 123 bytes")))
+        if not self.claim_close():
+            return void
+        return self._send((encode_close(chunk.code, reason),))
+
+    def write_all(self, chunks: Sequence[Chunk]) -> Effect[None, SocketError]:
+        if self.close_sent:
+            return void
+        return self._send(tuple(map(encode_message, chunks)))
+
+    def close(self, exit: Exit[object, object]) -> Effect[None]:
+        if not self.claim_close():
+            return void
+        return self._send((encode_close(close_code(exit)),)).catch_all(lambda _: void)
+
+
+def web_socket(exchange: Exchange, max_message_size: int) -> SocketService:
+    """The socket of a WebSocket upgrade request."""
+    opened = Latch(False)
+    current: list[_Session] = []
+
+    def handshake() -> Effect[_Session, SocketError]:
+        conn = exchange.conn
+        if exchange.upgraded is not None or exchange.headers_sent:
+            return fail(_closed_before_upgrade())
+        exchange.headers_sent = True
+        exchange.close_after = True
+        try:
+            head = _switching_head(exchange)
+        except _Refused as refused:
+            written = conn.write((refused.response(),)).catch_all(lambda _: void)
+            return written.flat_map(lambda _: sync(conn.end_writing)).flat_map(
+                lambda _: fail(_closed_before_upgrade())
+            )
+        exchange.upgraded = _SWITCHED
+        session = _Session(exchange, max_message_size)
+        return conn.write((head,)).map_error(lambda _: _closed_before_upgrade()).as_(session)
+
+    def release(session: _Session, exit: Exit[object, object]) -> Effect[None]:
+        opened.close_unsafe()
+        current.clear()
+        return session.close(exit)
+
+    def open_reader(session: _Session) -> Reader:
+        current.append(session)
+        opened.open_unsafe()
+        return Reader(session.pull())
+
+    reader = acquire_release(suspend(handshake), release).map(open_reader)
+
+    def write(chunk: Chunk | CloseEvent) -> Effect[None, SocketError]:
+        return opened.when_open(suspend(lambda: current[0].write(chunk)))
+
+    def write_all(chunks: Sequence[Chunk]) -> Effect[None, SocketError]:
+        return opened.when_open(suspend(lambda: current[0].write_all(chunks)))
+
+    writer: Effect[Writer, Never, Scope] = succeed(Writer(write, write_all))
+    return make(reader, writer)