headgate 0.2.0__py3-none-any.whl

headgate/_options.py

@@ -0,0 +1,358 @@
+# Copyright (c) 2026 Khaled Elsayed
+# SPDX-License-Identifier: MIT
+
+"""The option objects an :class:`~headgate.AsyncBaseClient` is configured with, beyond its URL, rate and auth.
+
+* :class:`TransportPolicy`: what the transport stack does for every request (retries, the deadline, the breaker,
+  per-path rates, the cache, idempotency keys, per-route overrides).
+* :class:`HttpOptions`: how a request is sent and its response read (the network transport and its TLS, pool and proxy,
+  headers, timeouts, redirects, the body cap, cookies, observers, middleware).
+* :class:`Timeouts`, :class:`IdempotencyKeys` and :class:`Route`, the values those hold.
+
+Every one is frozen, slotted and keyword-only, validates on construction so a bad configuration fails where it is
+written, and freezes the collections it is given.
+"""
+
+from __future__ import annotations
+
+import math
+import uuid
+from dataclasses import KW_ONLY, dataclass, field, replace
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final
+
+import httpx2
+
+from headgate._body import DEFAULT_MAX_RESPONSE_BYTES
+from headgate._core import origin_of, origin_text
+from headgate._failures import DEFAULT_REQUEST_ID_HEADERS
+from headgate.breaker import BreakerPolicy
+from headgate.retry import RetryPolicy
+
+if TYPE_CHECKING:
+    import ssl
+    from collections.abc import Callable, Iterable, Mapping
+
+    from headgate._core import Observer
+    from headgate._middleware import Middleware
+    from headgate._responses import RefusalCheck
+    from headgate.cache import HttpCache
+    from headgate.ratelimit import RateRule
+    from headgate.security import Resolver
+
+__all__ = [
+    "DEFAULT_LIMITS",
+    "KEYED_METHODS",
+    "HttpOptions",
+    "IdempotencyKeys",
+    "Route",
+    "Timeouts",
+    "TransportPolicy",
+    "httpx_timeout",
+    "match_route",
+    "reject_upstream_httpx",
+]
+
+#: The connection pool of the default network transport: 10 connections, 5 kept alive. The rate budget is the
+#: throttle; the pool only has to be wide enough never to bind first, and at 300 ms per request 10 connections sustain
+#: ~33 req/s. A client budgeted above that passes wider ``limits``.
+DEFAULT_LIMITS: Final = httpx2.Limits(max_connections=10, max_keepalive_connections=5)
+
+#: The proxy URL schemes httpx2 can speak (``socks5`` through the ``httpx2[socks]`` extra).
+_PROXY_SCHEMES: Final = frozenset({"http", "https", "socks5", "socks5h"})
+
+#: The schemes a trusted origin may name.
+_ORIGIN_SCHEMES: Final = frozenset({"http", "https"})
+
+
+def reject_upstream_httpx(value: object, *, what: str) -> None:
+    """Raise ``TypeError`` when *value* comes from upstream ``httpx``, which headgate does not support.
+
+    An ``httpx`` transport, auth or event hook passes no ``isinstance`` check against httpx2's classes but fails later
+    and obscurely, so the mistake is named where it is made: by the module of any class in the value's MRO, or of the
+    value itself for a function.
+    """
+    modules = [cls.__module__ for cls in type(value).__mro__]
+    modules.append(str(getattr(value, "__module__", "")))
+    if any(module == "httpx" or module.startswith("httpx.") for module in modules):
+        msg = (
+            f"{what} is an upstream httpx object ({type(value).__qualname__}); headgate is built on httpx2, so pass "
+            f"the httpx2 equivalent"
+        )
+        raise TypeError(msg)
+
+
+def _check_positive(name: str, value: float) -> None:
+    """Raise ``ValueError`` unless *value* is a positive number (NaN is not)."""
+    if not value > 0:
+        msg = f"{name} must be > 0 (got {value})"
+        raise ValueError(msg)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Timeouts:
+    """Per-phase timeouts in seconds: obtaining a connection from the pool, connecting, writing, reading."""
+
+    connect: float = 10.0
+    read: float = 30.0
+    write: float = 10.0
+    pool: float = 15.0
+
+    def __post_init__(self) -> None:
+        """Refuse a timeout that is not positive."""
+        for name in ("connect", "read", "write", "pool"):
+            _check_positive(name, getattr(self, name))
+
+
+def httpx_timeout(client: Timeouts | float, call: Timeouts | float | None = None) -> httpx2.Timeout:
+    """Return the ``httpx2.Timeout`` for a request: the *client*'s timeouts with the *call*'s applied.
+
+    A bare number is a read timeout: on the client it keeps the other defaults, on a call the client's other values.
+    A ``Timeouts`` replaces all four.
+    """
+    base = client if isinstance(client, Timeouts) else Timeouts(read=client)
+    if call is not None:
+        base = call if isinstance(call, Timeouts) else replace(base, read=call)
+    return httpx2.Timeout(connect=base.connect, read=base.read, write=base.write, pool=base.pool)
+
+
+def _uuid4_text() -> str:
+    """Return a random UUID as text."""
+    return str(uuid.uuid4())
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class IdempotencyKeys:
+    """Stamp a fresh idempotency key on every POST and PATCH a client sends, once per call, under ``header``.
+
+    The key is stamped before the retry layer, so every attempt of one call carries the same key and the upstream can
+    recognise a re-send. A call can pass its own with ``idempotency_key=``, and a key the caller set in ``headers=``
+    is kept.
+    """
+
+    header: str = "Idempotency-Key"
+    factory: Callable[[], str] = field(default=_uuid4_text)
+
+    def __post_init__(self) -> None:
+        """Refuse an empty header name."""
+        if not self.header:
+            msg = "header must name a header"
+            raise ValueError(msg)
+
+
+#: The methods :class:`IdempotencyKeys` stamps: the ones RFC 9110 does not define as idempotent.
+KEYED_METHODS: Final = frozenset({"POST", "PATCH"})
+
+
+@dataclass(frozen=True, slots=True)
+class Route:
+    """Overrides for the requests whose URL path contains ``fragment``: their own retry policy and deadline.
+
+    Matched like :class:`~headgate.RateRule`, the first matching route winning, so an endpoint's retry and deadline
+    change in the client's configuration rather than in the endpoint method. ``None`` keeps the policy's own value; a
+    call's ``retry=`` and ``deadline=`` win over both.
+    """
+
+    fragment: str
+    _: KW_ONLY
+    retry: RetryPolicy | None = None
+    deadline: float | None = None
+
+    def __post_init__(self) -> None:
+        """Refuse an empty fragment, which would match every path, and a deadline that is not positive."""
+        if not self.fragment:
+            msg = "fragment must not be empty"
+            raise ValueError(msg)
+        if self.deadline is not None:
+            _check_positive("deadline", self.deadline)
+
+
+def match_route(routes: tuple[Route, ...], path: str) -> Route | None:
+    """Return the first of *routes* whose fragment occurs in *path*, or ``None``."""
+    return next((route for route in routes if route.fragment in path), None)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TransportPolicy:
+    """What the transport stack does for every request.
+
+    * ``retry``: how a call is re-sent (:class:`~headgate.RetryPolicy`). A call opts out with ``retry=False``.
+    * ``deadline``: seconds the whole logical call may take, every attempt, wait and the body read included; ``None``
+      for none. Past it the call raises ``ClientTimeoutError``. A call overrides it with ``deadline=``.
+    * ``breaker``: the circuit breaker (:class:`~headgate.BreakerPolicy`), or ``None`` for none.
+    * ``rate_limits``: per-path budgets (:class:`~headgate.RateRule`): a request whose path matches a rule spends from
+      the first matching rule's budget instead of the client's.
+    * ``cache``: an opt-in :class:`~headgate.HttpCache`.
+    * ``idempotency``: stamp idempotency keys on POST and PATCH (:class:`IdempotencyKeys`), or ``None``.
+    * ``routes``: per-route retry and deadline overrides (:class:`Route`).
+    """
+
+    retry: RetryPolicy = field(default_factory=RetryPolicy)
+    deadline: float | None = 60.0
+    breaker: BreakerPolicy | None = field(default_factory=BreakerPolicy)
+    rate_limits: tuple[RateRule, ...] = ()
+    cache: HttpCache | None = None
+    idempotency: IdempotencyKeys | None = None
+    routes: tuple[Route, ...] = ()
+
+    def __post_init__(self) -> None:
+        """Refuse a deadline that is not positive and finite, and freeze the rule and route sequences."""
+        if self.deadline is not None and not (self.deadline > 0 and math.isfinite(self.deadline)):
+            msg = f"deadline must be > 0 and finite, or None (got {self.deadline})"
+            raise ValueError(msg)
+        object.__setattr__(self, "rate_limits", tuple(self.rate_limits))
+        object.__setattr__(self, "routes", tuple(self.routes))
+
+
+def _is_bare_origin(url: httpx2.URL) -> bool:
+    """Return whether *url* is an http(s) origin and nothing more: a host, no path, query, fragment or userinfo."""
+    return (
+        url.scheme in _ORIGIN_SCHEMES
+        and bool(url.host)
+        and url.path == "/"
+        and not (url.query or url.fragment or url.userinfo)
+    )
+
+
+def _normalized_origin(text: str) -> str:
+    """Return *text*, an ``http(s)://host[:port]`` origin, normalised; ``ValueError`` for anything else."""
+    try:
+        url = httpx2.URL(text)
+    except httpx2.InvalidURL:
+        url = None
+    if url is None or not _is_bare_origin(url):
+        msg = f"trusted origin must be an http(s) origin such as 'https://host[:port]' (got {text!r})"
+        raise ValueError(msg)
+    return origin_text(origin_of(url))
+
+
+#: What a proxy URL's error calls a URL httpx2 cannot parse, or one with no scheme.
+_MALFORMED = "malformed"
+
+
+def _check_proxy(proxy: str) -> None:
+    """Raise ``ValueError`` for a proxy URL httpx2 cannot use."""
+    try:
+        scheme = httpx2.URL(proxy).scheme or _MALFORMED
+    except httpx2.InvalidURL:
+        scheme = _MALFORMED
+    if scheme not in _PROXY_SCHEMES:
+        msg = f"proxy must be an http, https or socks5 URL (got a {scheme} URL)"
+        raise ValueError(msg)
+
+
+def _check_hooks(name: str, values: Iterable[object]) -> None:
+    """Refuse an upstream-httpx object among *values*."""
+    for value in values:
+        reject_upstream_httpx(value, what=f"an entry of {name}")
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class HttpOptions:
+    """How a request is sent and its response read.
+
+    * ``transport``: a custom network transport (``headgate.curl.AsyncCurlTransport``, a test double), which then owns
+      its own TLS, pool and proxy: ``ssl_context``, ``limits`` and ``proxy`` configure only the default transport, an
+      HTTP/2 ``httpx2.AsyncHTTPTransport``.
+    * ``headers``: sent on every request; a call's ``headers=`` wins a clash.
+    * ``timeout``: :class:`Timeouts`, or a number for the read timeout alone.
+    * ``follow_redirects`` and ``max_redirects``: the client's default, which a call can override, and the hop limit.
+      httpx2 counts an auth flow's hops in the same history as redirects, so each request a
+      :class:`~headgate.CachedTokenAuth` mint sends, and a 401 it re-mints after, uses one of ``max_redirects`` too.
+    * ``max_response_bytes``: the cap on a buffered body, counted after decompression.
+    * ``refusal_check``: a per-host check run on every final response (see :data:`~headgate.RefusalCheck`). It sees
+      only the final response, so a refusal written on a followed hop would be skipped, and it therefore requires
+      ``follow_redirects=False``.
+    * ``trusted_origins``: origins besides ``base_url``'s that credentials may be sent to (``"https://host[:port]"``);
+      every other origin gets the request without them (see :mod:`headgate.security`).
+    * ``allow_private_hosts``: ``False`` turns on the SSRF guard: a request to a host that is, or resolves to, an
+      address that is not globally routable raises :class:`~headgate.errors.ForbiddenDestinationError`, on every hop,
+      and the default transport connects only to the address it checked. It cannot be combined with ``proxy``, since a
+      proxy resolves the target itself; over a custom ``transport`` every hop is checked but not pinned.
+    * ``resolver``: how the SSRF guard resolves names (:data:`~headgate.security.Resolver`), the event loop's
+      ``getaddrinfo`` by default; only with ``allow_private_hosts=False``.
+    * ``user_agent``: a product token sent before headgate's own (``"<user_agent> headgate/<v> httpx2/<v>
+      python/<v>"``); without one, headgate's alone.
+    * ``cookies``: ``True`` keeps httpx2's cookie jar; by default no cookie a response sets is stored or sent back.
+    * ``request_id_headers``: the response headers a ``ClientError``'s ``request_id`` is read from, first match wins.
+    * ``observers``: receive the client's events (see :mod:`headgate.events`).
+    * ``middleware``: run once per attempt, in order, inside the retry layer (see :class:`~headgate.Middleware`).
+
+    Nothing is read from the environment: no proxy variable, no ``.netrc``, no ``SSL_CERT_FILE``.
+    """
+
+    transport: httpx2.AsyncBaseTransport | None = None
+    headers: Mapping[str, str] = field(default_factory=dict)
+    timeout: Timeouts | float = field(default_factory=Timeouts)
+    follow_redirects: bool = True
+    max_redirects: int = 5
+    max_response_bytes: int = DEFAULT_MAX_RESPONSE_BYTES
+    refusal_check: RefusalCheck | None = None
+    trusted_origins: frozenset[str] = frozenset()
+    allow_private_hosts: bool = True
+    resolver: Resolver | None = None
+    ssl_context: ssl.SSLContext | None = None
+    limits: httpx2.Limits = field(default_factory=lambda: DEFAULT_LIMITS)
+    proxy: str | None = None
+    user_agent: str | None = None
+    cookies: bool = False
+    request_id_headers: tuple[str, ...] = DEFAULT_REQUEST_ID_HEADERS
+    observers: tuple[Observer, ...] = ()
+    middleware: tuple[Middleware, ...] = ()
+
+    def __post_init__(self) -> None:
+        """Validate every option, and freeze and normalise the collections."""
+        self._check_transport()
+        if not isinstance(self.timeout, Timeouts):
+            _check_positive("timeout", self.timeout)
+        _check_positive("max_response_bytes", self.max_response_bytes)
+        if self.max_redirects < 0:
+            msg = f"max_redirects must be >= 0 (got {self.max_redirects})"
+            raise ValueError(msg)
+        self._check_guard()
+        if self.refusal_check is not None and self.follow_redirects:
+            msg = "an HttpOptions with a refusal_check= must set follow_redirects=False"
+            raise ValueError(msg)
+        if self.user_agent is not None and (not self.user_agent or not self.user_agent.isprintable()):
+            msg = "user_agent must be a non-empty, printable product token"
+            raise ValueError(msg)
+        names = tuple(name.lower() for name in self.request_id_headers)
+        if not all(names):
+            msg = "request_id_headers must name headers"
+            raise ValueError(msg)
+        _check_hooks("observers", self.observers)
+        _check_hooks("middleware", self.middleware)
+        object.__setattr__(self, "headers", MappingProxyType(dict(self.headers)))
+        object.__setattr__(self, "trusted_origins", frozenset(map(_normalized_origin, self.trusted_origins)))
+        object.__setattr__(self, "request_id_headers", names)
+        object.__setattr__(self, "observers", tuple(self.observers))
+        object.__setattr__(self, "middleware", tuple(self.middleware))
+
+    def _check_guard(self) -> None:
+        """Refuse a resolver without the SSRF guard, and the guard with a proxy, which would resolve past it."""
+        if self.allow_private_hosts and self.resolver is not None:
+            msg = "resolver= is used only by the SSRF guard, so it needs allow_private_hosts=False"
+            raise ValueError(msg)
+        if not self.allow_private_hosts and self.proxy is not None:
+            msg = "allow_private_hosts=False cannot be combined with proxy=: the proxy resolves the target itself"
+            raise ValueError(msg)
+
+    def _check_transport(self) -> None:
+        """Refuse an upstream-httpx transport, and default-transport settings next to a custom transport."""
+        if self.transport is None:
+            if self.proxy is not None:
+                _check_proxy(self.proxy)
+            return
+        reject_upstream_httpx(self.transport, what="transport")
+        configured = [
+            name
+            for name, is_set in (
+                ("ssl_context", self.ssl_context is not None),
+                ("limits", self.limits != DEFAULT_LIMITS),
+                ("proxy", self.proxy is not None),
+            )
+            if is_set
+        ]
+        if configured:
+            msg = f"{', '.join(configured)} configure the default transport, so they cannot be combined with transport="
+            raise ValueError(msg)

headgate/_redirects.py

@@ -0,0 +1,86 @@
+# Copyright (c) 2026 Khaled Elsayed
+# SPDX-License-Identifier: MIT
+
+"""What a followed redirect may keep and re-send: two response hooks every client installs.
+
+* :func:`drop_followed_redirect_body`: the body of a redirect httpx2 follows is dropped unread.
+* :func:`refuse_unreplayable_redirect`: a redirect that would send a one-shot request body a second time is not
+  followed; its 3xx is the call's final response.
+
+Both are installed by :func:`headgate._stack.build_http_client`; the call's :class:`~headgate._core.CallContext` tells
+them whether this request's redirects are being followed. Where a redirect may take a request, and what credentials it
+may carry there, is :mod:`headgate.security`'s.
+"""
+
+from __future__ import annotations
+
+from typing import Final
+
+import httpx2
+
+from headgate._core import CallContext
+from headgate.retry import REPLAYABLE_BODY
+
+__all__ = ["drop_followed_redirect_body", "refuse_unreplayable_redirect"]
+
+#: The redirects httpx2 turns into a body-less GET for a POST (RFC 9110 section 15.4.2 and 15.4.3 allow it).
+_POST_BECOMES_GET: Final = frozenset({httpx2.codes.MOVED_PERMANENTLY, httpx2.codes.FOUND})
+
+
+async def drop_followed_redirect_body(response: httpx2.Response) -> None:
+    """Discard the body of a redirect httpx2 is about to follow (a response hook).
+
+    httpx2 reads a followed redirect's body in full inside its own redirect loop, before the next hop, and so below
+    :func:`~headgate._body.read_capped`, which only ever sees the final response. A 302 whose body decompressed far
+    past ``max_response_bytes`` would otherwise be held in memory, kept alive in the response history. Nothing reads a
+    followed redirect's body, so it is replaced with an empty stream rather than capped. A redirect that is not followed
+    is the final response, and keeps its body for ``read_capped``. httpx2 hands the call's extensions, and so its
+    context, to every hop.
+
+    An ``AsyncClient`` always hands its hooks an async stream; the ``isinstance`` check only narrows the type.
+    """
+    stream = response.stream
+    context = CallContext.find(response.request)
+    if (
+        response.has_redirect_location
+        and context is not None
+        and context.follow_redirects
+        and isinstance(stream, httpx2.AsyncByteStream)
+    ):
+        await stream.aclose()
+        response.stream = httpx2.ByteStream(b"")
+
+
+def _resends_body(response: httpx2.Response) -> bool:
+    """Return whether httpx2 follows the redirect *response* by sending its request's body again.
+
+    It does for every redirect but a 303, which becomes a GET, and a 301 or 302 answering a POST, which httpx2 turns
+    into a GET as browsers do.
+    """
+    status = response.status_code
+    if status == httpx2.codes.SEE_OTHER:
+        return False
+    return not (status in _POST_BECOMES_GET and response.request.method == "POST")
+
+
+async def refuse_unreplayable_redirect(response: httpx2.Response) -> None:
+    """Stop httpx2 following a redirect that would send a one-shot request body again (a response hook).
+
+    A ``content=`` given as an async iterable is consumed by the first send (:data:`~headgate.retry.REPLAYABLE_BODY`),
+    so following a 307 or 308 would fail inside httpx2 with ``StreamConsumed``. The ``Location`` is removed instead, as
+    :func:`~headgate.security.refuse_https_downgrade` does: the 3xx reaches the call as its final answer, raised through
+    the status ladder unless the call declares it, with a note saying why. A body given as a factory is built afresh for
+    the next hop, so its redirects are followed.
+    """
+    if not response.has_redirect_location:
+        return
+    context = CallContext.find(response.request)
+    if context is None or not context.follow_redirects or context.get(REPLAYABLE_BODY) is not False:
+        return
+    if not _resends_body(response):
+        return
+    del response.headers["Location"]
+    context.add_note(
+        f"did not follow the {response.status_code} redirect: it asks for the request body again, and the body was a "
+        "one-shot stream (pass content= a factory to make it replayable)"
+    )

headgate/_request_body.py

@@ -0,0 +1,81 @@
+# Copyright (c) 2026 Khaled Elsayed
+# SPDX-License-Identifier: MIT
+
+"""A call's request body: raw ``content=`` (bytes, a stream, or a stream factory) and multipart ``files=``.
+
+A streamed body is either replayable or not, and the retry layer has to know which: a stream given as an iterable is
+consumed by the first send, so the call is marked non-replayable (:data:`~headgate.retry.REPLAYABLE_BODY`) and never
+re-sent; a stream given as a factory is built afresh, by calling the factory once, each time the request is sent.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Callable, Mapping, Sequence
+from typing import IO, override
+
+import httpx2
+
+__all__ = ["Body", "BodyFactory", "UploadFiles", "check_one_body", "request_content"]
+
+type BodyFactory = Callable[[], AsyncIterable[bytes]]
+"""Builds a fresh request body stream; called once for each send of the request."""
+
+type Body = bytes | AsyncIterable[bytes] | BodyFactory
+"""A raw request body: bytes, a one-shot stream, or a factory of streams (which keeps the call retryable)."""
+
+type _FileContent = IO[bytes] | bytes | str
+type _UploadFile = (
+    _FileContent
+    | tuple[str | None, _FileContent]
+    | tuple[str | None, _FileContent, str | None]
+    | tuple[str | None, _FileContent, str | None, Mapping[str, str]]
+)
+
+type UploadFiles = Mapping[str, _UploadFile] | Sequence[tuple[str, _UploadFile]]
+"""Multipart files, as httpx2 takes them: a file (or bytes), optionally with a filename, content type and headers."""
+
+
+class _Replayed(httpx2.AsyncByteStream):
+    """A request body built from its factory each time it is sent, so every attempt sends the whole body."""
+
+    def __init__(self, factory: BodyFactory) -> None:
+        """Build the body from *factory* on every send."""
+        self._factory = factory
+
+    @override
+    async def __aiter__(self) -> AsyncIterator[bytes]:
+        """Yield a fresh stream from the factory, closing it even when the send stops part-way."""
+        chunks = aiter(self._factory())
+        try:
+            async for chunk in chunks:
+                yield chunk
+        finally:
+            if isinstance(chunks, AsyncGenerator):
+                await chunks.aclose()
+
+
+def request_content(body: Body | None) -> tuple[bytes | AsyncIterable[bytes] | None, bool]:
+    """Return what httpx2 is handed as ``content=`` for *body*, and whether the request can be sent again.
+
+    Bytes and a factory are replayable; any other async iterable is consumed by the first send and is not. A value
+    that is both callable and iterable counts as an iterable.
+    """
+    if body is None or isinstance(body, bytes):
+        return body, True
+    if isinstance(body, AsyncIterable):
+        return body, False
+    return _Replayed(body), True
+
+
+def check_one_body(*, content: Body | None, json: object, data: object, files: UploadFiles | None) -> None:
+    """Raise ``ValueError`` for body options httpx2 would silently drop one of.
+
+    ``content=`` is the whole body, so it combines with nothing; ``files=`` is a multipart body, which takes form
+    fields from ``data=`` but cannot carry ``json=``.
+    """
+    if content is not None and (json is not None or data is not None or files is not None):
+        msg = "content= is the whole request body, so it cannot be combined with json=, data= or files="
+        raise ValueError(msg)
+    if files is not None and json is not None:
+        msg = "files= sends a multipart body, which cannot carry json=; pass form fields as data="
+        raise ValueError(msg)