pyfect-dev 0.1.0__py3-none-any.whl

pyfect/http/live_server/_head.py

@@ -0,0 +1,194 @@
+"""Request heads (RFC 9112): the request line, the header fields and how
+the body is framed. Parsing is strict, as Node's ``llhttp`` is: a head it
+cannot read is a ``HeadError`` with the status to answer. Values are
+trimmed of optional whitespace (SP and HTAB) only, the request-target is
+one of the four forms of RFC 9112 §3.2, and ``Host`` is an authority."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Literal
+from urllib.parse import urlsplit
+
+from pyfect._digits import parse_digits
+from pyfect.http._authority import ROOT, Authority, OriginForm, parse_authority, parse_origin_form
+from pyfect.http.headers import Headers
+
+_TOKEN = re.compile(rb"[!#$%&'*+\-.^_`|~0-9A-Za-z]+\Z")
+_VERSION = re.compile(rb"HTTP/(\d)\.(\d)\Z")
+_OWS = " \t"
+_FIELD_VALUE = re.compile(rb"[\t\x20-\x7e\x80-\xff]*\Z")
+_TARGET = re.compile(rb"[\x21-\x7e\x80-\xff]+\Z")
+
+
+class HeadError(Exception):
+    """A head that cannot be read: ``status`` is the response it gets."""
+
+    def __init__(self, status: int, message: str) -> None:
+        super().__init__(message)
+        self.status = status
+
+
+@dataclass(frozen=True, slots=True)
+class Length:
+    """A body of ``size`` bytes (``0``: none)."""
+
+    size: int
+
+
+@dataclass(frozen=True, slots=True)
+class Chunked:
+    """A chunked body."""
+
+
+type Framing = Length | Chunked
+
+
+@dataclass(frozen=True, slots=True)
+class RequestHead:
+    """``target`` is the path and query (or ``*``; ``/`` for a
+    ``CONNECT``); ``authority`` is where it was sent, from ``Host``, a
+    ``CONNECT``'s target or an absolute-form target (which replaces
+    ``host``)."""
+
+    method: str
+    target: OriginForm | Literal["*"]
+    authority: Authority | None
+    version: tuple[int, int]
+    headers: Headers
+    framing: Framing
+    keep_alive: bool
+    expect: Literal["continue", "none"]
+    upgrade: str | None
+
+
+def _items(value: str) -> list[str]:
+    """The lowercase items of a comma-separated field value, trimmed of
+    optional whitespace (SP and HTAB) only: ``str.strip()`` would also
+    take NBSP and NEL, which a strict parser keeps as part of the item."""
+    return [item.strip(_OWS).lower() for item in value.split(",")]
+
+
+def _connection_tokens(headers: dict[str, str]) -> set[str]:
+    return set(_items(headers.get("connection", "")))
+
+
+def _framing(headers: dict[str, str], version: tuple[int, int]) -> Framing:
+    transfer_encoding = headers.get("transfer-encoding")
+    content_length = headers.get("content-length")
+    if transfer_encoding is not None:
+        if content_length is not None:
+            raise HeadError(400, "Content-Length with Transfer-Encoding")
+        codings = _items(transfer_encoding)
+        if version < (1, 1) or codings[-1] != "chunked" or codings.count("chunked") != 1:
+            raise HeadError(400, "Invalid Transfer-Encoding")
+        if codings != ["chunked"]:
+            raise HeadError(501, "Unsupported Transfer-Encoding")
+        return Chunked()
+    if content_length is None:
+        return Length(0)
+    values = set(_items(content_length))
+    if len(values) != 1:
+        raise HeadError(400, "Invalid Content-Length")
+    length = parse_digits(values.pop())
+    if length is None:
+        raise HeadError(400, "Invalid Content-Length")
+    return Length(length)
+
+
+def _add(headers: dict[str, str], name: str, value: str) -> None:
+    previous = headers.get(name)
+    if previous is None:
+        headers[name] = value
+    elif name == "cookie":
+        headers[name] = f"{previous}; {value}"
+    elif name == "host":
+        raise HeadError(400, "Duplicate Host")
+    else:
+        headers[name] = f"{previous}, {value}"
+
+
+def _authority(text: str) -> Authority:
+    authority = parse_authority(text)
+    if authority is None:
+        raise HeadError(400, "Invalid Host")
+    return authority
+
+
+def _target(
+    method: str, target: str, headers: dict[str, str]
+) -> tuple[OriginForm | Literal["*"], Authority | None]:
+    """The path and query (or ``*``) and the authority the request was sent
+    to: ``Host``, a ``CONNECT``'s target, or an absolute-form target, which
+    replaces ``host`` (RFC 9112 §3.2.2)."""
+    origin = parse_origin_form(target)
+    if origin is not None or target == "*":
+        host = _authority(headers["host"]) if headers.get("host") else None
+        return origin or "*", host
+    if method == "CONNECT":
+        return ROOT, _authority(target)
+    parts = urlsplit(target)
+    if parts.scheme.lower() not in ("http", "https") or not target[len(parts.scheme) :].startswith(
+        "://"
+    ):
+        raise HeadError(400, "Invalid request-target")
+    authority = _authority(parts.netloc)
+    headers["host"] = authority
+    path = parts.path or "/"
+    origin = parse_origin_form(f"{path}?{parts.query}" if parts.query else path)
+    if origin is None:
+        raise HeadError(400, "Invalid request-target")
+    return origin, authority
+
+
+def parse_field_line(line: bytes) -> tuple[str, str]:
+    """A header or trailer field line (RFC 9112 §5): its lowercase name
+    and its value without optional whitespace."""
+    name, colon, value = line.partition(b":")
+    if not colon or not _TOKEN.match(name):
+        raise HeadError(400, "Invalid header field")
+    value = value.strip(b" \t")
+    if not _FIELD_VALUE.match(value):
+        raise HeadError(400, "Invalid header value")
+    return name.decode("ascii").lower(), value.decode("latin-1")
+
+
+def parse_head(data: bytes) -> RequestHead:
+    """The head ``data`` holds: the lines before the blank one, without
+    their line ends."""
+    lines = data.split(b"\r\n")
+    parts = lines[0].split(b" ")
+    if len(parts) != 3 or not _TOKEN.match(parts[0]) or not _TARGET.match(parts[1]):
+        raise HeadError(400, "Invalid request line")
+    version_match = _VERSION.match(parts[2])
+    if version_match is None:
+        raise HeadError(400, "Invalid HTTP version")
+    version = (int(version_match[1]), int(version_match[2]))
+    if version not in ((1, 0), (1, 1)):
+        raise HeadError(505, "HTTP Version Not Supported")
+    headers: dict[str, str] = {}
+    for line in lines[1:]:
+        _add(headers, *parse_field_line(line))
+    if version == (1, 1) and "host" not in headers:
+        raise HeadError(400, "Missing Host")
+    method = parts[0].decode("ascii")
+    target, authority = _target(method, parts[1].decode("latin-1"), headers)
+    framing = _framing(headers, version)
+    tokens = _connection_tokens(headers)
+    keep_alive = "keep-alive" in tokens if version == (1, 0) else "close" not in tokens
+    expect_header = headers.get("expect")
+    if expect_header is not None and _items(expect_header) != ["100-continue"]:
+        raise HeadError(417, "Expectation Failed")
+    upgrade = headers.get("upgrade") if "upgrade" in tokens else None
+    return RequestHead(
+        method,
+        target,
+        authority,
+        version,
+        Headers._unsafe(headers),  # pyright: ignore[reportPrivateUsage]
+        framing,
+        keep_alive,
+        "continue" if expect_header is not None and version == (1, 1) else "none",
+        upgrade,
+    )

pyfect/http/live_server/_inbound.py

@@ -0,0 +1,156 @@
+"""The bytes a connection receives, read by one fiber at a time: request
+heads, then body bytes, with what was read too far kept for the next
+read."""
+
+from __future__ import annotations
+
+import re
+import time
+from collections.abc import Callable
+
+from pyfect._core import Effect
+from pyfect.duration import Seconds
+from pyfect.effect import fail, succeed, suspend
+from pyfect.http.live_server._timers import timeout_option
+from pyfect.socket._connection import Connection
+from pyfect.socket._errors import SocketError, is_clean_close, read_error
+
+
+class HeadTooLarge(Exception):
+    """The head did not end within the limit."""
+
+
+class BareLineFeed(Exception):
+    """A head line ended in LF without CR (``llhttp`` answers 400 at once)."""
+
+
+_BARE_LF = re.compile(rb"(?<!\r)\n")
+
+
+class RequestTimeout(Exception):
+    """The request did not arrive whole within the request timeout."""
+
+
+class Inbound:
+    """``read`` gives ``b""`` once the peer ended the stream cleanly; a
+    connection that failed fails the read. Once ``deadline`` is set (a
+    ``time.monotonic()``), a wait for bytes past it fails the read."""
+
+    __slots__ = ("_conn", "_ended", "_pending", "deadline")
+
+    def __init__(self, conn: Connection) -> None:
+        self._conn = conn
+        self._pending = bytearray()
+        self._ended = False
+        self.deadline: float | None = None
+
+    @property
+    def has_pending(self) -> bool:
+        return bool(self._pending)
+
+    def take_pending(self) -> bytes:
+        """What was received and not read yet (handed to an upgrade)."""
+        data = bytes(self._pending)
+        self._pending.clear()
+        return data
+
+    def _fill(self) -> Effect[bool, SocketError]:
+        """Wait for more bytes; ``False`` at the end of the stream."""
+        if self._ended:
+            return succeed(False)
+
+        def received(chunks: tuple[bytes, ...]) -> bool:
+            for chunk in chunks:
+                self._pending += chunk
+            return True
+
+        def ended(error: SocketError) -> Effect[bool, SocketError]:
+            if not is_clean_close(error):
+                return fail(error)
+            self._ended = True
+            return succeed(False)
+
+        pulled = self._conn.pull_into(received).catch_all(ended)
+        if self.deadline is None:
+            return pulled
+        left = self.deadline - time.monotonic()
+        if left <= 0:
+            return fail(read_error(RequestTimeout()))
+        return timeout_option(pulled, Seconds(left)).flat_map(
+            lambda more: fail(read_error(RequestTimeout())) if more is None else succeed(more.value)
+        )
+
+    def wait_data(self) -> Effect[None, SocketError]:
+        """Wait until some bytes are received (or the stream ends)."""
+        if self._pending or self._ended:
+            return succeed(None)
+        return self._fill().as_void()
+
+    def read_head(
+        self, limit: int
+    ) -> Effect[bytes | None, SocketError | HeadTooLarge | BareLineFeed]:
+        """The next head without its blank line; ``None`` at a clean end
+        before any of it. Blank lines (CRLF or LF, as ``llhttp``) before a
+        head are skipped (RFC 9112 §2.2)."""
+
+        def step() -> Effect[bytes | None, SocketError | HeadTooLarge | BareLineFeed]:
+            pending = self._pending
+            while pending.startswith((b"\n", b"\r\n")):
+                del pending[: 1 if pending.startswith(b"\n") else 2]
+            end = self._pending.find(b"\r\n\r\n")
+            bare = _BARE_LF.search(self._pending, 0, len(self._pending) if end == -1 else end)
+            if bare is not None:
+                return fail(BareLineFeed())
+            if end != -1:
+                if end > limit:
+                    return fail(HeadTooLarge())
+                head = bytes(self._pending[:end])
+                del self._pending[: end + 4]
+                return succeed(head)
+            if len(self._pending) > limit:
+                return fail(HeadTooLarge())
+            return self._fill().flat_map(lambda more: suspend(step) if more else self._ended_head())
+
+        return suspend(step)
+
+    def _ended_head(self) -> Effect[bytes | None, SocketError]:
+        if self._pending:
+            return fail(read_error(ConnectionError("the connection ended within a head")))
+        return succeed(None)
+
+    def read[A, E](
+        self, maximum: int, take: Callable[[bytes], Effect[A, E]]
+    ) -> Effect[A, SocketError | E]:
+        """``take`` of up to ``maximum`` bytes, waiting for some (``b""`` at
+        the end); the bytes are taken and handed over in one step."""
+
+        def step() -> Effect[A, SocketError | E]:
+            if self._pending or self._ended:
+                data = bytes(self._pending[:maximum])
+                del self._pending[:maximum]
+                return take(data)
+            return self._fill().flat_map(lambda _: suspend(step))
+
+        return suspend(step)
+
+    def read_line[A, E](
+        self, limit: int, take: Callable[[bytes | None], Effect[A, E]]
+    ) -> Effect[A, SocketError | E]:
+        """``take`` of the next line without its CRLF (``None`` at the end,
+        past ``limit``, or at a line ending in a bare LF, which ``llhttp``
+        fails at once), taken and handed over in one step."""
+
+        def step() -> Effect[A, SocketError | E]:
+            pending = self._pending
+            lf = pending.find(b"\n")
+            if lf != -1:
+                if lf == 0 or pending[lf - 1] != 0x0D:
+                    return take(None)
+                line = bytes(pending[: lf - 1])
+                del pending[: lf + 1]
+                return take(line)
+            if len(self._pending) > limit or self._ended:
+                return take(None)
+            return self._fill().flat_map(lambda _: suspend(step))
+
+        return suspend(step)

pyfect/http/live_server/_listener.py

@@ -0,0 +1,211 @@
+"""Listening for HTTP connections on the socket loop, and the graceful
+shutdown Node's server does when its scope closes: stop accepting, close
+idle connections, let the busy ones finish (they close once idle for the
+keep-alive timeout) for ``graceful_shutdown_timeout``, then interrupt them
+and abort the connections that do not end at once."""
+
+from __future__ import annotations
+
+import asyncio
+import ssl
+import threading
+from collections.abc import Callable
+from typing import Any, Never
+
+from pyfect._core import Effect
+from pyfect.duration import Seconds
+from pyfect.effect import gen, suspend, sync
+from pyfect.exit import Success
+from pyfect.http.live_server._timers import timeout_option
+from pyfect.net.net_address import (
+    SocketAddress,
+    inet_address_from_native_unsafe,
+    unix_path_address,
+)
+from pyfect.scope import CloseableScope, Scope, acquire_release, fork_in
+from pyfect.socket._connection import Connection, Signal
+from pyfect.socket._loop import get_loop, run
+from pyfect.socket.server import SocketServerError, SocketServerOpenError
+
+type Listen = Callable[[Callable[[], Connection]], Any]
+
+ABORT_GRACE_PERIOD = Seconds(0.5)
+"""How long the requests still running when the graceful shutdown timeout
+ends may take to stop once interrupted before their connections are
+aborted."""
+
+
+class ConnectionState:
+    """A connection the server tracks: ``idle`` between requests."""
+
+    __slots__ = ("conn", "idle")
+
+    def __init__(self, conn: Connection) -> None:
+        self.conn = conn
+        self.idle = True
+
+
+class Listener:
+    """Connections accepted and being served."""
+
+    __slots__ = ("_closed", "_pending", "_serving", "address", "lock", "shutting_down", "signal")
+
+    def __init__(self) -> None:
+        self.lock = threading.Lock()
+        self.signal = Signal(self.lock)
+        self._pending: list[Connection] = []
+        self._serving: set[ConnectionState] = set()
+        self._closed = False
+        self.shutting_down = False
+        self.address: SocketAddress | None = None
+
+    # --- loop side ----------------------------------------------------------
+
+    def made(self, conn: Connection) -> None:
+        with self.lock:
+            closed = self._closed
+            if not closed:
+                self._pending.append(conn)
+            waiters = self.signal.notify_locked()
+        if closed:
+            conn.close(graceful=False)
+        for wake in waiters:
+            wake()
+
+    # --- fiber side -----------------------------------------------------------
+
+    def _take(self) -> list[ConnectionState]:
+        with self.lock:
+            taken, self._pending = self._pending, []
+            states = [ConnectionState(conn) for conn in taken]
+            self._serving.update(states)
+        return states
+
+    def done(self, state: ConnectionState) -> None:
+        with self.lock:
+            self._serving.discard(state)
+            waiters = self.signal.notify_locked()
+        for wake in waiters:
+            wake()
+
+    def run(
+        self, serve: Callable[[ConnectionState], Effect[None]], shutdown_timeout: Seconds
+    ) -> Effect[Never]:
+        """Serve every connection, each on a fiber of a scope closed, after
+        the graceful shutdown, when this is interrupted."""
+
+        def loop(scope: CloseableScope) -> Effect[Never]:
+            def fork_taken():
+                for state in self._take():
+                    conn = state.conn
+                    fiber = yield from fork_in(serve(state), scope)
+                    fiber.add_observer_unsafe(lambda _, s=state, c=conn: self._release(s, c))
+
+            hand_over = gen(fork_taken).uninterruptible()
+
+            def forever():
+                while True:
+                    yield from self.signal.wait(lambda: bool(self._pending))
+                    yield from hand_over
+
+            return gen(forever)
+
+        def close_all(scope: CloseableScope) -> Effect[None]:
+            """As Node's ``closeAllConnections``: interrupt the requests still
+            running, and abort the connections that do not end within the
+            grace period (a write waiting on a client that stopped reading,
+            which an interrupt cannot end)."""
+            serving = self.serving()
+
+            def abort() -> None:
+                for conn in serving:
+                    conn.close(graceful=False)
+
+            def body():
+                closing = yield from scope.close(Success(None)).fork()
+                if (yield from timeout_option(closing.await_exit(), ABORT_GRACE_PERIOD)) is None:
+                    abort()
+                    yield from closing.await_exit()
+
+            return gen(body)
+
+        def body():
+            scope = CloseableScope()
+            closed = self.shutdown(shutdown_timeout).and_then(suspend(lambda: close_all(scope)))
+            return (yield from loop(scope).ensuring(closed))
+
+        return gen(body)
+
+    def _release(self, state: ConnectionState, conn: Connection) -> None:
+        conn.close(graceful=True)
+        self.done(state)
+
+    def serving(self) -> list[Connection]:
+        """The connections being served."""
+        with self.lock:
+            return [state.conn for state in self._serving]
+
+    def close(self) -> None:
+        """Close connections made from now on."""
+        with self.lock:
+            self._closed = True
+            pending, self._pending = self._pending, []
+        for conn in pending:
+            conn.close(graceful=False)
+
+    def shutdown(self, timeout: Seconds) -> Effect[None]:
+        """Stop serving new requests: close the idle connections and wait
+        up to ``timeout`` for the busy ones."""
+
+        def start() -> None:
+            with self.lock:
+                self.shutting_down = True
+                self._closed = True
+                pending, self._pending = self._pending, []
+                idle = [state.conn for state in self._serving if state.idle]
+            for conn in pending + idle:
+                conn.close(graceful=True)
+
+        drained = timeout_option(self.signal.wait_until(lambda: not self._serving), timeout)
+        return sync(start).flat_map(lambda _: drained).as_void()
+
+
+def _address(server: asyncio.Server) -> SocketAddress:
+    name: Any = server.sockets[0].getsockname()
+    if isinstance(name, str):
+        return unix_path_address(name)
+    return inet_address_from_native_unsafe(name[0], name[1])
+
+
+def listen(
+    listen: Listen, tls: ssl.SSLContext | None
+) -> Effect[Listener, SocketServerError, Scope]:
+    """A listener that stops accepting when the scope closes; each run
+    listens anew."""
+    return suspend(lambda: _listen(Listener(), listen, tls))
+
+
+def _listen(
+    listener: Listener, listen: Listen, tls: ssl.SSLContext | None
+) -> Effect[Listener, SocketServerError, Scope]:
+
+    async def start() -> asyncio.Server:
+        return await listen(lambda: Connection(listener.made, server_tls=tls))
+
+    def stop(server: asyncio.Server, _exit: object) -> Effect[None]:
+        async def close() -> None:
+            loop = get_loop()
+            for listening in server.sockets:
+                loop.remove_reader(listening.fileno())
+            await asyncio.sleep(0)
+            await asyncio.sleep(0)
+            server.close()
+
+        return run(close).ignore().flat_map(lambda _: sync(listener.close))
+
+    def opened(server: asyncio.Server) -> Listener:
+        listener.address = _address(server)
+        return listener
+
+    starting = run(start).map_error(lambda cause: SocketServerError(SocketServerOpenError(cause)))
+    return acquire_release(starting, stop).map(opened)