xtr-http-client 4.0.2__py3-none-any.whl

xtr_http_client/internal/httpx_clients.py

@@ -0,0 +1,203 @@
+"""The backend clients a request is actually opened on, one per way of connecting."""
+
+from __future__ import annotations
+
+import ssl
+import urllib.request
+from typing import TYPE_CHECKING, Final, final
+
+import httpx
+
+from xtr_http_client.exception import InvalidArgumentError
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+    from typing import TypeAlias
+
+    _Key: TypeAlias = tuple[object, ...]
+
+__all__ = ["HttpxClients"]
+
+# What `timeout` of `None` means: long enough for a slow server, short enough
+# to free the connection. The contract names the number; this is where it is
+# turned into one.
+_IDLE_TIMEOUT: Final = 60.0
+
+_HTTP_1_1: Final = (None, "1.1")
+_HTTP_2: Final = ("2", "2.0")
+
+
+@final
+class HttpxClients:
+    """One backend client per set of connection settings, opened when first asked for.
+
+    Two requests can only share a connection pool when they would be happy
+    with each other's connections. A proxy, a trust store, a client
+    certificate, the protocol version and the deadlines all decide what a
+    connection *is*, so each combination of them gets a pool of its own, made
+    the first time something asks for it — which is what keeps a client that
+    is never used from opening anything.
+
+    The environment's proxy settings are part of that too. The backend reads
+    them once, when a pool is made, so they are read here as well and a
+    change in them asks for a different pool rather than being ignored.
+
+    Each pool also remembers the addresses that requests pinned their hosts
+    to, for as long as the pool lives: a host reached at an address once is
+    reached at it again, without asking a resolver a second time.
+
+    ``max_connections`` is the backend's pool-wide ceiling, not a per-host
+    one: it counts every connection the pool holds, whichever host each of
+    them goes to.
+    """
+
+    __slots__ = ("_clients", "_max_connections", "_resolved")
+
+    def __init__(self, max_connections: int) -> None:
+        """Open nothing yet.
+
+        Args:
+            max_connections: How many connections one pool may hold at once,
+                counted across every host it reaches.
+        """
+        self._max_connections = max_connections
+        self._clients: dict[_Key, httpx.AsyncClient] = {}
+        self._resolved: dict[_Key, dict[str, str]] = {}
+
+    def for_options(
+        self,
+        options: Mapping[str, object],
+    ) -> tuple[httpx.AsyncClient, dict[str, str]]:
+        """Return the pool a request made with ``options`` belongs to.
+
+        Args:
+            options: The request's options, merged over the client's
+                defaults.
+
+        Returns:
+            The pool's client, and the addresses it has had hosts pinned to.
+
+        Raises:
+            InvalidArgumentError: When the HTTP version asked for is not one
+                this client can send.
+        """
+        key = _key(options)
+        client = self._clients.get(key)
+        if client is None:
+            client = _build(options, self._max_connections)
+            self._clients[key] = client
+            self._resolved[key] = {}
+
+        return client, self._resolved[key]
+
+    async def aclose(self) -> None:
+        """Close every pool, and with it every connection any of them holds."""
+        clients = list(self._clients.values())
+        self._clients.clear()
+        self._resolved.clear()
+
+        for client in clients:
+            await client.aclose()
+
+
+def _key(options: Mapping[str, object]) -> _Key:
+    """What makes two requests able to share a pool."""
+    return (
+        options.get("proxy"),
+        options.get("verify_peer"),
+        options.get("verify_host"),
+        options.get("cafile"),
+        options.get("capath"),
+        options.get("local_cert"),
+        options.get("local_pk"),
+        options.get("passphrase"),
+        _wants_http2(options.get("http_version")),
+        options.get("timeout"),
+        options.get("max_connect_duration"),
+        tuple(sorted(urllib.request.getproxies().items())),
+    )
+
+
+def _build(options: Mapping[str, object], max_connections: int) -> httpx.AsyncClient:
+    """Open the pool ``options`` asks for."""
+    proxy = options.get("proxy")
+    timeout = options.get("timeout")
+    connect = options.get("max_connect_duration")
+
+    try:
+        return httpx.AsyncClient(
+            trust_env=True,
+            follow_redirects=False,
+            http2=_wants_http2(options.get("http_version")),
+            limits=httpx.Limits(max_connections=max_connections),
+            timeout=httpx.Timeout(
+                float(timeout) if isinstance(timeout, (int, float)) else _IDLE_TIMEOUT,
+                connect=float(connect) if isinstance(connect, (int, float)) and connect else None,
+            ),
+            proxy=str(proxy) if proxy else None,
+            verify=_verification(options),
+        )
+    except ImportError as error:
+        # The HTTP/2 framing is an optional install, and asking for a version
+        # that was never installed is an argument this client cannot honour.
+        raise InvalidArgumentError(
+            'Option "http_version" of "2" needs the "http2" extra: install xtr-http-client[http2].'
+        ) from error
+
+
+def _wants_http2(version: object) -> bool:
+    """Whether ``version`` asks for HTTP/2.
+
+    Raises:
+        InvalidArgumentError: When it asks for a version this client cannot
+            send. HTTP/1.0 is one of them: nothing here can be made to leave
+            out what 1.1 added.
+    """
+    if version in _HTTP_1_1:
+        return False
+    if version in _HTTP_2:
+        return True
+
+    raise InvalidArgumentError(
+        f'Invalid HTTP version "{version}", only "1.1" and "2" can be asked for.'
+    )
+
+
+def _verification(options: Mapping[str, object]) -> ssl.SSLContext | bool:
+    """How the peer's certificate is to be checked, as the backend takes it.
+
+    ``True`` and ``False`` are the two plain answers — check with the
+    platform's trust store, or check nothing. Anything in between is a
+    context, because that is the only place another trust store, a client
+    certificate or a name that is not to be matched can be said.
+    """
+    if not options.get("verify_peer", True):
+        return False
+
+    verify_host = bool(options.get("verify_host", True))
+    cafile = _text(options.get("cafile"))
+    capath = _text(options.get("capath"))
+    local_cert = _text(options.get("local_cert"))
+
+    if verify_host and cafile is None and capath is None and local_cert is None:
+        return True
+
+    context = ssl.create_default_context(cafile=cafile, capath=capath)
+    # Order matters: the hostname check cannot be turned off while the
+    # certificate is still being verified against a name.
+    context.check_hostname = verify_host
+    context.verify_mode = ssl.CERT_REQUIRED
+
+    if local_cert is not None:
+        context.load_cert_chain(
+            local_cert,
+            _text(options.get("local_pk")),
+            _text(options.get("passphrase")),
+        )
+
+    return context
+
+
+def _text(value: object) -> str | None:
+    """``value`` as text, or ``None`` when it was left out."""
+    return value if isinstance(value, str) else None

xtr_http_client/internal/httpx_errors.py

@@ -0,0 +1,63 @@
+"""What a failure of the backend transfer means in this package's own terms.
+
+A caller holding the client interface has never heard of the backend's own
+exceptions, so none of them may reach one. The translation is here, in one
+place, because the same failure can surface while the head is being read and
+again while the body is: the head is read where the request is made, the body
+wherever the caller asks for it.
+"""
+
+from __future__ import annotations
+
+import httpx
+
+from xtr_http_client.exception import (
+    InvalidArgumentError,
+    TransportError,
+    TransportTimeoutError,
+)
+
+__all__ = ["timed_out", "transport_error"]
+
+
+def transport_error(error: Exception, url: str) -> TransportError:
+    """Return the error to raise for a failed transfer, naming the URL it failed for.
+
+    Three kinds are told apart, because a caller does different things about
+    them: running out of time is worth trying again, a URL or a scheme the
+    backend cannot make a request from is worth correcting, and everything
+    else is the network.
+
+    Args:
+        error: What the backend raised.
+        url: The URL the transfer was for, kept in the message so a failure
+            reported far from the call still says what it was about.
+
+    Returns:
+        The error to raise, which the caller is to chain onto ``error``.
+    """
+    said = str(error) or type(error).__name__
+    reason = f'{said} for "{url}".'
+
+    if isinstance(error, httpx.TimeoutException):
+        return TransportTimeoutError(reason)
+    if isinstance(error, (httpx.UnsupportedProtocol, httpx.InvalidURL)):
+        return InvalidArgumentError(reason)
+
+    return TransportError(reason)
+
+
+def timed_out(url: str) -> TransportTimeoutError:
+    """Return the error for a transfer that outlasted ``max_duration``.
+
+    The deadline is this package's own rather than the backend's — the whole
+    request and response are measured, which nothing in the backend counts —
+    so there is no backend error to translate.
+
+    Args:
+        url: The URL the transfer was for.
+
+    Returns:
+        The error to raise, which the caller is to chain onto the deadline.
+    """
+    return TransportTimeoutError(f'Maximum duration reached for "{url}".')

xtr_http_client/internal/httpx_redirect.py

@@ -0,0 +1,63 @@
+"""Where an answer points, and whether following it stays where it started."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, cast
+
+if TYPE_CHECKING:
+    import httpx
+
+__all__ = ["next_hop", "same_origin"]
+
+
+def next_hop(response: httpx.Response) -> httpx.Request | None:
+    """Return the request that following ``response`` would make.
+
+    Args:
+        response: The answer to read a redirect off.
+
+    Returns:
+        The next request, or ``None`` when there is nowhere to go — either
+        because the answer is not a redirect, or because where it points is
+        not somewhere a request can be made to.
+    """
+    following = response.next_request
+    if following is None:
+        return None
+    location = cast("str", response.headers.get("location", ""))
+    if _names_no_host(location) or not following.url.host:
+        return None
+
+    return following
+
+
+def same_origin(one: httpx.URL, other: httpx.URL) -> bool:
+    """Tell whether two URLs name the same scheme, host and port.
+
+    Args:
+        one: A URL.
+        other: The URL to compare it with.
+
+    Returns:
+        ``True`` when a connection to either would be a connection to the
+        same place.
+    """
+    return (one.scheme, one.host, one.port) == (other.scheme, other.host, other.port)
+
+
+def _names_no_host(location: str) -> bool:
+    """Whether a Location names an authority and then leaves it empty.
+
+    ``//?foo=bar`` says "another host, the same scheme" and names no host, so
+    there is nowhere for it to point; a relative reference such as ``..``
+    names no authority at all, which is not the same thing — it is resolved
+    against the URL it arrived on.
+    """
+    if not location.startswith("//"):
+        return False
+
+    authority = location[2:]
+    for delimiter in ("/", "?", "#"):
+        authority = authority.partition(delimiter)[0]
+
+    return authority == ""

xtr_http_client/internal/httpx_request.py

@@ -0,0 +1,272 @@
+"""What an outgoing request is assembled from, before a connection is opened.
+
+The request is authoritative and the options fill what it leaves out, so each
+of its four parts — the URL, the headers, the body and the address the host is
+reached at — is worked out from the two of them together. That is one decision
+per part, made here, away from the exchange and the redirect chain that follow.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterable, Mapping, Sequence
+from typing import TYPE_CHECKING, Final, NamedTuple, cast
+
+import httpx
+
+from xtr_http_client.exception import InvalidArgumentError, TransportError
+
+from .debug_type import debug_type
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+    from xtr_http_message_contracts import RequestInterface
+
+__all__ = [
+    "Payload",
+    "assemble_headers",
+    "assemble_payload",
+    "base_uri",
+    "carries",
+    "checked_body",
+    "header_values",
+    "pin_host",
+    "target_url",
+]
+
+# Quoted by the contract rather than described, so it is written out once and
+# never composed.
+_BOTH_BODIES: Final = (
+    'Define either the request body or the "body" or "json" option, setting both is not supported.'
+)
+
+# What a body of bytes, text or pieces is taken to be when nobody said: the
+# only shape a request body has without a content type to read it by.
+_FORM: Final = "application/x-www-form-urlencoded"
+
+_NO_BODY: Final = (None, b"", "")
+
+
+class Payload(NamedTuple):
+    """The one body of a request, in whichever of three shapes it is to be sent."""
+
+    content: bytes | str | AsyncIterator[bytes] | None
+    form: Mapping[str, object] | None
+    document: object
+
+
+def target_url(request: RequestInterface, options: Mapping[str, object]) -> httpx.URL:
+    """The URL to request: the request's own, resolved and with ``query`` merged in."""
+    url = httpx.URL(str(request.get_uri()))
+
+    base = options.get("base_uri")
+    if url.is_relative_url and base is not None:
+        url = httpx.URL(base_uri(base)).join(url)
+
+    query = options.get("query")
+    if query:
+        url = url.copy_merge_params(cast("Mapping[str, str]", query))
+
+    return url
+
+
+def base_uri(base: object) -> str:
+    """The URI a relative URL is resolved against.
+
+    A sequence of them is for a client that retries, which moves along it
+    from one attempt to the next; this one makes a single attempt, so it
+    resolves against the first.
+    """
+    if isinstance(base, str):
+        return base
+    if isinstance(base, Sequence) and base:
+        first = base[0]
+        if isinstance(first, str):
+            return first
+
+    raise InvalidArgumentError(
+        f'Option "base_uri" must be a URL or a list of them, "{debug_type(base)}" given.'
+    )
+
+
+def assemble_headers(
+    request: RequestInterface,
+    options: Mapping[str, object],
+) -> list[tuple[str, str]]:
+    """Every header to send: the request's, then the names it left out.
+
+    The ``Host`` the request worked out from its own URI is dropped, because
+    the URL actually requested may not be that one — a relative URI resolved
+    against a base, or a host pinned to an address, both move it — and a
+    header nobody asked for must not be the one that decides where a request
+    lands. One the caller set by hand is kept and sent exactly as given.
+    """
+    headers = [
+        (name, value)
+        for name, values in request.get_headers().items()
+        for value in values
+        if not _is_implied_host(name, value, request)
+    ]
+
+    for name, value in _mapping(options.get("headers")).items():
+        if carries(headers, name):
+            continue
+        headers += header_values(name, value)
+
+    bearer = options.get("auth_bearer")
+    if isinstance(bearer, str) and bearer and not carries(headers, "authorization"):
+        headers.append(("Authorization", f"Bearer {bearer}"))
+
+    return headers
+
+
+def _is_implied_host(name: str, value: str, request: RequestInterface) -> bool:
+    """Whether a header is the ``Host`` a request filled in from its own URI."""
+    if name.lower() != "host":
+        return False
+
+    uri = request.get_uri()
+    host = uri.get_host()
+    port = uri.get_port()
+
+    return value == (host if port is None else f"{host}:{port}")
+
+
+async def assemble_payload(
+    request: RequestInterface,
+    options: Mapping[str, object],
+    headers: list[tuple[str, str]],
+) -> Payload:
+    """What to send as the body, from the request or from the options, never both.
+
+    A request carrying a body has already said what to send, so a ``body`` or
+    ``json`` option alongside it is two answers to one question and is
+    refused rather than one of them being picked.
+
+    Raises:
+        InvalidArgumentError: When the request carries a body and an option
+            gives another.
+    """
+    carried = await request.get_body().get_contents()
+    if carried:
+        if options.get("body") not in _NO_BODY or options.get("json") is not None:
+            raise InvalidArgumentError(_BOTH_BODIES)
+        _imply_form(headers)
+
+        return Payload(carried, None, None)
+
+    return _asked_for(options, headers)
+
+
+def _asked_for(options: Mapping[str, object], headers: list[tuple[str, str]]) -> Payload:
+    """What the options alone say to send, for a request that carried no body.
+
+    A mapping is a form, which the backend writes out; bytes, text and an
+    iterable are a body nobody labelled, so they are said to be a form here;
+    and a JSON document labels itself.
+    """
+    document = options.get("json")
+    if document is not None:
+        return Payload(None, None, document)
+
+    body = options.get("body")
+    if isinstance(body, Mapping):
+        return Payload(None, cast("Mapping[str, object]", body), None)
+    if isinstance(body, (bytes, str)) and body:
+        _imply_form(headers)
+
+        return Payload(body, None, None)
+    if isinstance(body, AsyncIterable):
+        _imply_form(headers)
+
+        return Payload(checked_body(body), None, None)
+
+    return Payload(None, None, None)
+
+
+async def checked_body(source: AsyncIterable[object]) -> AsyncIterator[bytes]:
+    """``source``, with a piece that is not bytes refused before it reaches the wire.
+
+    Raises:
+        TransportError: When the body produces anything other than bytes.
+            The request fails rather than the backend tripping over it with
+            an error from somewhere a caller cannot place.
+    """
+    async for piece in source:
+        if not isinstance(piece, (bytes, bytearray)):
+            raise TransportError(
+                f'The request body produced "{debug_type(piece)}", which is not bytes.'
+            )
+        yield bytes(piece)
+
+
+def pin_host(
+    url: httpx.URL,
+    options: Mapping[str, object],
+    resolved: dict[str, str],
+    headers: list[tuple[str, str]],
+) -> tuple[httpx.URL, dict[str, str]]:
+    """The URL to connect to, with the host swapped for the address it is pinned to.
+
+    Pinning is what makes a request reach an address that has already been
+    checked, so nothing can change under it between the check and the
+    connection. The host survives the swap in two places that still need it:
+    the ``Host`` header, which is what the server routes on, and the name the
+    TLS handshake asks for.
+    """
+    resolved.update(_text_mapping(options.get("resolve")))
+
+    host = url.raw_host.decode("ascii")
+    address = resolved.get(host)
+    if address is None:
+        return url, {}
+
+    _replace(headers, "Host", host if url.port is None else f"{host}:{url.port}")
+
+    return url.copy_with(host=address), {"sni_hostname": host}
+
+
+def carries(headers: Sequence[tuple[str, str]], name: str) -> bool:
+    """Whether a header of that name is already there, matched without case."""
+    wanted = name.lower()
+
+    return any(present.lower() == wanted for present, _ in headers)
+
+
+def header_values(name: str, value: object) -> list[tuple[str, str]]:
+    """One header name and every value given for it."""
+    if isinstance(value, str):
+        return [(name, value)]
+    if isinstance(value, Sequence):
+        return [(name, str(one)) for one in value]
+
+    raise InvalidArgumentError(
+        f'Option "headers" must map a name to a string or a list of them, '
+        f'"{debug_type(value)}" given for "{name}".'
+    )
+
+
+def _replace(headers: list[tuple[str, str]], name: str, value: str) -> None:
+    """Make ``name`` hold ``value`` and nothing else."""
+    wanted = name.lower()
+    headers[:] = [entry for entry in headers if entry[0].lower() != wanted]
+    headers.append((name, value))
+
+
+def _imply_form(headers: list[tuple[str, str]]) -> None:
+    """Say what an unlabelled body is, since a server cannot read one unlabelled."""
+    if not carries(headers, "content-type"):
+        headers.append(("Content-Type", _FORM))
+
+
+def _mapping(value: object) -> Mapping[str, object]:
+    """``value`` as a mapping, or an empty one when it was left out."""
+    if isinstance(value, Mapping):
+        return cast("Mapping[str, object]", value)
+
+    return {}
+
+
+def _text_mapping(value: object) -> dict[str, str]:
+    """``value`` as a mapping of text to text, keeping only the entries that are."""
+    return {name: entry for name, entry in _mapping(value).items() if isinstance(entry, str)}