pyfect-dev 0.1.0__py3-none-any.whl

pyfect/http/url_params.py

@@ -0,0 +1,304 @@
+"""URL query parameters (Effect's ``UrlParams``): ordered ``(name, value)``
+pairs, where a value may be ``Redacted`` to keep it out of logs while it is
+still sent.
+
+``from_input`` takes a mapping or pairs; values are coerced the way
+JavaScript's ``String`` does (``True`` is ``"true"``, ``None`` is
+``"null"``, ``1.0`` is ``"1"``), a sequence value adds one pair per item and
+a nested mapping adds ``a[b]`` keys::
+
+    params = from_input({"q": "effect", "page": 2}).append("tag", "a")
+    to_string(params)   # "q=effect&page=2&tag=a"
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from typing import cast, final
+from urllib.parse import parse_qsl, quote_plus, unquote
+
+from pyfect._js import js_number, text_encode
+from pyfect.context import Reference
+from pyfect.http.headers import Pattern, is_redacted_name
+from pyfect.redacted import Redacted
+from pyfect.redacted import value as redacted_value
+
+type Value = str | Redacted[str]
+type Param = tuple[str, Value]
+type Coercible = str | int | float | bool | Redacted[str] | None
+
+type CoercibleRecord = Mapping[str, Coercible | Sequence[Coercible] | CoercibleRecord]
+
+type Input = UrlParams | CoercibleRecord | Iterable[tuple[str, Coercible]]
+
+
+def _unwrap(value: Value) -> str:
+    return value if isinstance(value, str) else redacted_value(value)
+
+
+@final
+class UrlParams:
+    """Ordered query parameters. Iterating gives the pairs; ``repr`` hides
+    the values of ``CurrentRedactedNames`` and of ``Redacted`` values."""
+
+    __slots__ = ("params",)
+
+    def __init__(self, params: Iterable[Param] = ()) -> None:
+        self.params: tuple[Param, ...] = tuple(params)
+
+    def __iter__(self) -> Iterator[Param]:
+        return iter(self.params)
+
+    def __len__(self) -> int:
+        return len(self.params)
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, UrlParams):
+            return NotImplemented
+        return self.params == other.params
+
+    def __hash__(self) -> int:
+        return hash(self.params)
+
+    def __repr__(self) -> str:
+        return f"UrlParams({self.redact(current_redacted_names())!r})"
+
+    def get_all(self, key: str) -> list[str]:
+        return [_unwrap(v) for k, v in self.params if k == key]
+
+    def get_first(self, key: str) -> str | None:
+        return next((_unwrap(v) for k, v in self.params if k == key), None)
+
+    def get_last(self, key: str) -> str | None:
+        return next((_unwrap(v) for k, v in reversed(self.params) if k == key), None)
+
+    def set(self, key: str, value: Coercible) -> UrlParams:
+        """Replace every pair named ``key`` with one, at the end."""
+        return UrlParams((*(p for p in self.params if p[0] != key), (key, _coerce(value))))
+
+    def transform(self, f: Callable[[tuple[Param, ...]], Iterable[Param]]) -> UrlParams:
+        return UrlParams(f(self.params))
+
+    def set_all(self, input: Input) -> UrlParams:
+        """The pairs of ``input``, then the pairs of ``self`` whose names
+        ``input`` does not set."""
+        params = from_input(input).params
+        keys = {k for k, _ in params}
+        return UrlParams((*params, *(p for p in self.params if p[0] not in keys)))
+
+    def append(self, key: str, value: Coercible) -> UrlParams:
+        return UrlParams((*self.params, (key, _coerce(value))))
+
+    def append_all(self, input: Input) -> UrlParams:
+        return UrlParams((*self.params, *from_input(input).params))
+
+    def remove(self, key: str) -> UrlParams:
+        return UrlParams(p for p in self.params if p[0] != key)
+
+    def to_record(self) -> dict[str, str | list[str]]:
+        """Each name to its value, or to its values when repeated."""
+        out: dict[str, str | list[str]] = {}
+        for key, raw in self.params:
+            value = _unwrap(raw)
+            current = out.get(key)
+            if current is None:
+                out[key] = value
+            elif isinstance(current, str):
+                out[key] = [current, value]
+            else:
+                current.append(value)
+        return out
+
+    def redact(self, key: Pattern | Sequence[Pattern]) -> list[tuple[str, Value]]:
+        """The pairs, with plain values of the names matching ``key`` hidden
+        in ``Redacted``."""
+        patterns: Sequence[Pattern] = (key,) if isinstance(key, str | re.Pattern) else key
+        return [
+            (
+                name,
+                Redacted(value)
+                if isinstance(value, str) and is_redacted_name(name, patterns)
+                else value,
+            )
+            for name, value in self.params
+        ]
+
+    def to_json(self) -> dict[str, object]:
+        return {
+            "_id": "UrlParams",
+            "params": {k: str(v) for k, v in self.redact(current_redacted_names())},
+        }
+
+
+EMPTY: UrlParams = UrlParams()
+
+
+def make(params: Iterable[Param]) -> UrlParams:
+    return UrlParams(params)
+
+
+def _coerce(value: Coercible) -> Value:
+    if isinstance(value, Redacted | str):
+        return value
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    return js_number(value)
+
+
+def from_input(input: Input) -> UrlParams:
+    if isinstance(input, UrlParams):
+        return input
+    return UrlParams(
+        (key if isinstance(key, str) else f"{key[0]}[{']['.join(key[1:])}]", value)
+        for key, value in _nested(input)
+    )
+
+
+def _nested(
+    input: CoercibleRecord | Iterable[tuple[str, Coercible]],
+) -> list[tuple[str | list[str], Value]]:
+    entries: Iterable[tuple[str, object]] = (
+        cast("Mapping[str, object]", input).items() if isinstance(input, Mapping) else input
+    )
+    out: list[tuple[str | list[str], Value]] = []
+    for key, value in entries:
+        if isinstance(value, list | tuple):
+            out.extend((key, _coerce(item)) for item in cast("Sequence[Coercible]", value))
+        elif isinstance(value, Mapping):
+            for k, v in _nested(cast("CoercibleRecord", value)):
+                out.append(([key, *([k] if isinstance(k, str) else k)], v))
+        else:
+            out.append((key, _coerce(cast("Coercible", value))))
+    return out
+
+
+def _encode_component(text: str) -> str:
+    """``application/x-www-form-urlencoded`` byte serialization, as
+    ``URLSearchParams`` writes it: only ``*-._`` and alphanumerics stay."""
+    return quote_plus(text_encode(text), safe="*").replace("~", "%7E")
+
+
+def to_string(input: Input) -> str:
+    """The parameters as a query string, without the ``?``; ``Redacted``
+    values are written out."""
+    return "&".join(
+        f"{_encode_component(k)}={_encode_component(_unwrap(v))}" for k, v in from_input(input)
+    )
+
+
+def parse(query: str) -> UrlParams:
+    """A query string (``?`` optional) as ``URLSearchParams`` reads it:
+    ``+`` is a space and bad escapes stay as written."""
+    return UrlParams(parse_qsl(query.removeprefix("?"), keep_blank_values=True))
+
+
+class CurrentRedactedNames(Reference[tuple[Pattern, ...]]):
+    """The parameter names whose values logs, spans and error messages hide."""
+
+    @classmethod
+    def default(cls) -> tuple[Pattern, ...]:
+        return ("AWSAccessKeyId", "Signature", "sig", "X-Goog-Signature")
+
+
+def current_redacted_names() -> tuple[Pattern, ...]:
+    """``CurrentRedactedNames`` of the running fiber, or the default."""
+    from pyfect._fiber import current_fiber
+
+    fiber = current_fiber()
+    return CurrentRedactedNames.default() if fiber is None else fiber.get_ref(CurrentRedactedNames)
+
+
+_BAD_ESCAPE = re.compile(r"%(?![0-9A-Fa-f]{2})")
+_AUTHORITY_START = re.compile(
+    r"^(?:(https?|ftp|wss?):[/\\]*|[a-z][a-z0-9+.-]*://|[/\\]{2,})", re.I | re.ASCII
+)
+_SPECIAL_AUTHORITY_END = re.compile(r"[/\\?#]")
+_AUTHORITY_END = re.compile(r"[/?#]")
+
+
+def _redact_userinfo(url: str) -> str:
+    # Normalize only to locate the userinfo; offsets keep the original spelling.
+    normalized: list[str] = []
+    offsets: list[int] = []
+    for i, ch in enumerate(url):
+        if ch in "\t\n\r" or (not normalized and ord(ch) <= 32):
+            continue
+        normalized.append(ch)
+        offsets.append(i)
+    text = "".join(normalized)
+    authority = _AUTHORITY_START.match(text)
+    if authority is None:
+        return url
+    start = len(authority[0])
+    special = authority[1] is not None or ":" not in authority[0]
+    rest = text[start:]
+    end = (_SPECIAL_AUTHORITY_END if special else _AUTHORITY_END).search(rest)
+    userinfo_end = (rest if end is None else rest[: end.start()]).rfind("@")
+    if userinfo_end == -1:
+        return url
+    return url[: offsets[start]] + "REDACTED:REDACTED" + url[offsets[start + userinfo_end] :]
+
+
+def _decode_query_key(key: str) -> str:
+    if "%" not in key and "+" not in key:
+        return key
+    if _BAD_ESCAPE.search(key):
+        return key
+
+    try:
+        return unquote(key.replace("+", " "), errors="strict")
+    except UnicodeDecodeError:
+        return key
+
+
+def _redact_query(query: str, names: Sequence[Pattern]) -> str:
+    def part(text: str) -> str:
+        index = text.find("=")
+        if index != -1 and is_redacted_name(_decode_query_key(text[:index]), names):
+            return text[: index + 1] + "REDACTED"
+        return text
+
+    return "&".join(part(p) for p in query.split("&"))
+
+
+def redact_url(url: str, redacted_names: Sequence[Pattern] | None = None) -> str:
+    """``url`` with its userinfo replaced by ``REDACTED:REDACTED`` and the
+    values of the redacted query parameters by ``REDACTED`` (by default
+    those of the running fiber's ``CurrentRedactedNames``)."""
+    names = current_redacted_names() if redacted_names is None else redacted_names
+    if "@" in url:
+        url = _redact_userinfo(url)
+    query_start = url.find("?")
+    if query_start == -1 or not names:
+        return url
+    hash_start = url.find("#")
+    if hash_start != -1 and hash_start < query_start:
+        return url
+    query_end = len(url) if hash_start == -1 else hash_start
+    return (
+        url[: query_start + 1]
+        + _redact_query(url[query_start + 1 : query_end], names)
+        + url[query_end:]
+    )
+
+
+__all__ = [
+    "EMPTY",
+    "Coercible",
+    "CoercibleRecord",
+    "CurrentRedactedNames",
+    "Input",
+    "Param",
+    "UrlParams",
+    "Value",
+    "current_redacted_names",
+    "from_input",
+    "is_redacted_name",
+    "make",
+    "parse",
+    "redact_url",
+    "to_string",
+]

pyfect/http/uvicorn_server.py

@@ -0,0 +1,119 @@
+"""An ``HttpServer`` on uvicorn (extra ``pyfect-dev[server]``; the counterpart
+of Effect's ``NodeHttpServer.layer``)::
+
+    main = server.serve(app).pipe(layer.provide(uvicorn_server.layer(port=8080)))
+
+The socket is bound when the server is made, so ``address`` is known (port
+0: any free port). uvicorn runs on the building fiber's asyncio loop under
+``run_main`` / ``run_async``, else on a loop thread shared by such servers
+and never stopped (a fiber a request forks into a longer-lived set runs on
+it, and must still be able to finish once the server is gone); each
+request runs on its own fiber on that loop, through ``pyfect.http.asgi``
+(streamed bodies, client aborts, WebSocket upgrades). Requests that arrive
+while no app is served get a 503. When the serving scope closes, uvicorn
+stops accepting and gives requests in flight ``graceful_shutdown_timeout``;
+then their fibers are interrupted and streamed bodies end where they are
+(ASGI cannot abort a started response quietly, so it ends complete).
+"""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+import collections.abc
+import contextlib
+from collections.abc import Mapping
+from typing import Any, NotRequired, Unpack
+
+from pyfect._extras import missing_extra
+
+try:
+    import uvicorn
+except ImportError as error:  # pragma: no cover - depends on the environment
+    raise missing_extra("pyfect.http.uvicorn_server", "server") from error
+
+from pyfect._core import Effect
+from pyfect.file_system import FileSystem
+from pyfect.http._asgi_listener import Dispatch, ListenOptions, graceful, make_listener
+from pyfect.http.client import HttpClient
+from pyfect.http.etag import Generator
+from pyfect.http.httpx_client import layer as httpx_layer
+from pyfect.http.live_server import layer_services
+from pyfect.http.live_server._listener import ABORT_GRACE_PERIOD
+from pyfect.http.platform import HttpPlatform
+from pyfect.http.server import HttpServer, HttpServerService, layer_test_client
+from pyfect.layer import Layer, merge, provide_merge
+from pyfect.layer import effect as layer_effect
+from pyfect.path import Path
+from pyfect.scope import Scope as ScopeTag
+from pyfect.socket._web_socket import DEFAULT_MAX_MESSAGE_SIZE
+from pyfect.socket.server import SocketServerError
+
+
+class Options(ListenOptions):
+    config: NotRequired[Mapping[str, Any]]
+    """More ``uvicorn.Config`` arguments (``http``, ``ws``, ``ws_max_size``,
+    ``proxy_headers``, ``timeout_keep_alive``, ``log_config``...); the app,
+    socket and lifespan are the server's."""
+
+
+class _Server(uvicorn.Server):
+    """uvicorn without its signal handlers: the program owns SIGINT and
+    SIGTERM (``run_main`` interrupts it, which closes the server's scope and
+    runs ``_Running.shut_down``). uvicorn's would shut it down on its own,
+    cancelling streamed responses at its timeout."""
+
+    @contextlib.contextmanager
+    def capture_signals(self) -> collections.abc.Generator[None]:
+        yield
+
+
+def _uvicorn(dispatch: Dispatch, options: Options) -> uvicorn.Server:
+    config: dict[str, Any] = {
+        "log_config": None,
+        "server_header": False,
+        "ws_max_size": DEFAULT_MAX_MESSAGE_SIZE,
+        **options.get("config", {}),
+    }
+    # uvicorn's own timeout only backs up ``_Running.shut_down``, which ends
+    # the requests first: its cancel is logged with a traceback.
+    backstop = graceful(options) + ABORT_GRACE_PERIOD
+    config.update(app=dispatch, lifespan="off", timeout_graceful_shutdown=backstop)
+    return _Server(uvicorn.Config(**config))
+
+
+def make(**options: Unpack[Options]) -> Effect[HttpServerService, SocketServerError, ScopeTag]:
+    """An ``HttpServer`` on uvicorn, listening on ``host:port`` until the
+    scope closes."""
+    return make_listener(options, lambda dispatch: _uvicorn(dispatch, options))
+
+
+def layer_server(**options: Unpack[Options]) -> Layer[HttpServer, SocketServerError]:
+    return layer_effect(HttpServer, make(**options))
+
+
+def layer(
+    **options: Unpack[Options],
+) -> Layer[HttpServer | Path | HttpPlatform | FileSystem | Generator, SocketServerError]:
+    """The server and the services its apps use (``live_server.layer_services``)."""
+    return merge(layer_server(**options), layer_services)
+
+
+def layer_test(
+    **options: Unpack[Options],
+) -> Layer[
+    HttpServer | Path | HttpPlatform | FileSystem | Generator | HttpClient, SocketServerError
+]:
+    """A server on a free loopback port, its services, and an ``HttpClient``
+    (httpx) sending to it (Effect's ``NodeHttpServer.layerTest``)."""
+    server = layer(**{"host": "127.0.0.1", **options})
+    return layer_test_client.pipe(provide_merge(httpx_layer)).pipe(provide_merge(server))
+
+
+__all__ = [
+    "Options",
+    "SocketServerError",
+    "layer",
+    "layer_server",
+    "layer_test",
+    "make",
+]

pyfect/http/web.py

@@ -0,0 +1,171 @@
+"""Requests and responses as asyncio code sees them (Effect's Web
+``Request`` and ``Response``): what ``web_handler`` handlers take and
+return, and what the ASGI adapter speaks.
+
+A body is ``bytes``, an async iterable of ``bytes`` (read once) or
+``None``. Header names are lowercase; repeated request headers are joined
+with ``", "``, and only ``set-cookie`` repeats in a response.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterable, AsyncIterator, Iterable, Mapping
+from typing import Any, cast, final
+
+from pyfect import _json_text
+from pyfect._js import text_encode
+from pyfect.http.abort import AbortSignal
+from pyfect.http.headers import Headers
+
+type WebBody = bytes | AsyncIterable[bytes] | None
+type HeaderLines = Mapping[str, str] | Iterable[tuple[str, str]]
+
+
+def _lines(headers: HeaderLines | None) -> tuple[tuple[str, str], ...]:
+    if headers is None:
+        return ()
+    pairs: Iterable[tuple[str, str]] = (
+        cast("Mapping[str, str]", headers).items() if isinstance(headers, Mapping) else headers
+    )
+    return tuple((key.lower(), value) for key, value in pairs)
+
+
+def _body_of(body: bytes | str | AsyncIterable[bytes] | None) -> WebBody:
+    return text_encode(body) if isinstance(body, str) else body
+
+
+async def _read(body: WebBody) -> bytes:
+    if body is None:
+        return b""
+    if isinstance(body, bytes):
+        return body
+    return b"".join([chunk async for chunk in body])
+
+
+async def _once(body: bytes | None) -> AsyncIterator[bytes]:
+    if body:
+        yield body
+
+
+class _Message:
+    __slots__ = ("_used", "body", "header_lines")
+
+    def __init__(
+        self, headers: HeaderLines | None, body: bytes | str | AsyncIterable[bytes] | None
+    ):
+        self.header_lines = _lines(headers)
+        self.body: WebBody = _body_of(body)
+        self._used = False
+
+    def get(self, name: str) -> str | None:
+        """Header ``name``, repeated values joined with ``", "``."""
+        name = name.lower()
+        values = [value for key, value in self.header_lines if key == name]
+        return ", ".join(values) if values else None
+
+    def has(self, name: str) -> bool:
+        name = name.lower()
+        return any(key == name for key, _ in self.header_lines)
+
+    async def read(self) -> bytes:
+        """The whole body (it can be read once)."""
+        if self._used:
+            raise RuntimeError("body already read")
+        self._used = True
+        return await _read(self.body)
+
+    async def buffer(self) -> bytes:
+        """Read the body into memory, so it can be read again (what Web
+        ``Response.clone`` allows)."""
+        if self._used:
+            raise RuntimeError("body already read")
+        if not (self.body is None or isinstance(self.body, bytes)):
+            self.body = await _read(self.body)
+        return self.body or b""
+
+    async def text(self) -> str:
+        return (await self.read()).decode("utf-8", "replace")
+
+    async def json(self) -> Any:
+        return _json_text.parse(await self.read())
+
+    def chunks(self) -> AsyncIterator[bytes]:
+        """The body, chunk by chunk."""
+
+        self._used = True
+        body = self.body
+        if body is None or isinstance(body, bytes):
+            return _once(body)
+        # The body itself, so closing the iterator closes it.
+        return aiter(body)
+
+
+@final
+class WebRequest(_Message):
+    """A request a server received: an absolute ``url``, and a ``signal``
+    that fires when the client goes away."""
+
+    __slots__ = ("method", "remote_address", "signal", "url")
+
+    def __init__(
+        self,
+        url: str,
+        *,
+        method: str = "GET",
+        headers: HeaderLines | None = None,
+        body: bytes | str | AsyncIterable[bytes] | None = None,
+        signal: AbortSignal | None = None,
+        remote_address: str | None = None,
+    ) -> None:
+        super().__init__(headers, body)
+        self.url = url
+        self.method = method.upper()
+        self.signal = signal
+        self.remote_address = remote_address
+
+    @property
+    def headers(self) -> Headers:
+        joined: dict[str, str] = {}
+        for key, value in self.header_lines:
+            joined[key] = f"{joined[key]}, {value}" if key in joined else value
+        return Headers(joined)
+
+
+@final
+class WebResponse(_Message):
+    """A response as sent."""
+
+    __slots__ = ("status", "status_text")
+
+    def __init__(
+        self,
+        body: bytes | str | AsyncIterable[bytes] | None = None,
+        *,
+        status: int = 200,
+        status_text: str = "",
+        headers: HeaderLines | None = None,
+    ) -> None:
+        lines = _lines(headers)
+        if isinstance(body, str) and not any(key == "content-type" for key, _ in lines):
+            lines = (*lines, ("content-type", "text/plain;charset=UTF-8"))
+        super().__init__(lines, body)
+        self.status = status
+        self.status_text = status_text
+
+    @classmethod
+    def json_of(cls, value: object, *, status: int = 200) -> WebResponse:
+        """A JSON response (Web ``Response.json``)."""
+        return cls(
+            text_encode(_json_text.stringify(value)),
+            status=status,
+            headers={"content-type": "application/json"},
+        )
+
+    def get_set_cookie(self) -> list[str]:
+        return [value for key, value in self.header_lines if key == "set-cookie"]
+
+    def __repr__(self) -> str:
+        return f"WebResponse(status={self.status}, headers={self.header_lines!r})"
+
+
+__all__ = ["HeaderLines", "WebBody", "WebRequest", "WebResponse"]