pathapi 0.1.0__py3-none-any.whl

pathapi/http/_response.py

@@ -0,0 +1,255 @@
+"""The response every transport returns, with the decoding they share."""
+
+from __future__ import annotations
+
+import errno as _errno
+import typing as _ty
+import zlib as _zlib
+
+from ._headers import Headers
+
+CHUNK = 64 * 1024
+"""Most bytes read from a transport's body in one call."""
+
+_STEP = 256 * 1024
+"""Most decoded bytes one decompression step produces. The compressed bytes
+that are left wait for the next step, so a body that expands a thousandfold
+is never held whole because its header said it was small."""
+
+
+class _Memory:
+    """A body that is already in memory."""
+
+    def __init__(self, data):
+        self._data = memoryview(bytes(data))
+        self._at = 0
+
+    def read(self, amount: int) -> bytes:
+        chunk = bytes(self._data[self._at : self._at + amount])
+        self._at += len(chunk)
+        return chunk
+
+    def close(self) -> None:
+        pass
+
+
+class _Inflater:
+    """Decompresses one `gzip` or `deflate` body in bounded steps. `deflate` is
+    read as zlib-wrapped data, or as raw data when the zlib header is absent,
+    which is what servers send under that name; concatenated gzip members are
+    one body."""
+
+    def __init__(self, gzip: bool):
+        self._gzip = gzip
+        self._object = None
+        self._input = b""
+        self._members = 0
+
+    @property
+    def has_input(self) -> bool:
+        return bool(self._input)
+
+    def feed(self, data: bytes) -> None:
+        self._input += data
+
+    def _new(self, data: bytes):
+        if self._gzip:
+            return _zlib.decompressobj(16 + _zlib.MAX_WBITS)
+        wrapped = len(data) < 2 or (
+            data[0] & 0x0F == 8 and ((data[0] << 8) | data[1]) % 31 == 0
+        )
+        return _zlib.decompressobj(_zlib.MAX_WBITS if wrapped else -_zlib.MAX_WBITS)
+
+    def step(self) -> bytes:
+        data = self._input
+        if self._object is None:
+            self._object = self._new(data)
+        try:
+            out = self._object.decompress(data, _STEP)
+        except _zlib.error:
+            if self._members:
+                # Bytes after the last complete member are not a body.
+                self._input, self._object = b"", None
+                return b""
+            kind = "gzip" if self._gzip else "deflate"
+            raise OSError(
+                _errno.EIO, f"the response body is not valid {kind} data"
+            ) from None
+        if self._object.eof:
+            self._members += 1
+            self._input = self._object.unused_data
+            self._object = None
+        else:
+            self._input = self._object.unconsumed_tail
+            if not out and len(self._input) >= len(data):
+                self._input = b""
+        return out
+
+    def finish(self) -> None:
+        if self._object is not None:
+            raise OSError(_errno.EIO, "the compressed response body ended early")
+
+
+def _inflater_for(headers: Headers) -> "_Inflater | None":
+    names = [
+        name.strip().lower()
+        for name in headers.get("Content-Encoding", "").split(",")
+        if name.strip().lower() not in ("", "identity")
+    ]
+    if len(names) != 1:
+        return None
+    if names[0] in ("gzip", "x-gzip"):
+        return _Inflater(gzip=True)
+    if names[0] == "deflate":
+        return _Inflater(gzip=False)
+    return None
+
+
+class HTTPResponse:
+    """One response: `status`, `reason`, `headers` (a read-only mapping that
+    ignores the case of a name), `url` (the URL the request went to) and the
+    body, read with `read()` or `iter_bytes()`. A context manager that closes.
+
+    The body is decoded when the response says it is `gzip` or `deflate`
+    encoded, unless the request set its own `Accept-Encoding`; `headers` are
+    the headers as received, so `Content-Length` counts the encoded bytes. A
+    body that ends before the length the server stated is `OSError(EIO)`, never
+    a shorter body that reads as complete.
+
+    A transport builds one with `body` as `bytes`, or as an object with
+    `read(amount) -> bytes` (empty at the end) and `close()`, and
+    `decode=False` for a body it already decoded or that was not asked for
+    in a coding."""
+
+    status: int
+    reason: str
+    headers: Headers
+    url: str
+
+    def __init__(
+        self,
+        status: int,
+        reason: str,
+        headers: "_ty.Mapping[str, str] | _ty.Iterable[tuple[str, str]]",
+        url: str,
+        body: "bytes | _ty.Any" = b"",
+        *,
+        decode: bool = True,
+    ):
+        self._closed = False
+        self._ended = False
+        self.status = int(status)
+        self.reason = reason or ""
+        self.headers = headers if isinstance(headers, Headers) else Headers(headers)
+        self.url = url
+        self._kept = False
+        if isinstance(body, (bytes, bytearray, memoryview)):
+            body = _Memory(body)
+            self._kept = True  # in memory: readable after close
+        self._source = body
+        self._inflater = _inflater_for(self.headers) if decode else None
+        self._leftover = b""
+
+    def __repr__(self) -> str:
+        return f"HTTPResponse({self.status}, {self.reason!r})"
+
+    def __enter__(self) -> "HTTPResponse":
+        return self
+
+    def __exit__(self, *exc_info) -> None:
+        self.close()
+
+    def __del__(self):
+        try:
+            self.close()
+        except Exception:
+            pass
+
+    def _check_open(self) -> None:
+        if self._closed and not self._kept:
+            raise ValueError("read of a closed response")
+
+    def _next(self, limit: int = CHUNK) -> bytes:
+        """The next decoded bytes of the body; empty at its end. Without a
+        coding, no more than `limit` bytes are asked of the transport."""
+        if self._ended:
+            return b""
+        self._check_open()
+        inflater = self._inflater
+        while True:
+            if inflater is not None and inflater.has_input:
+                out = inflater.step()
+                if out:
+                    return out
+                continue
+            out = self._source.read(CHUNK if inflater else min(CHUNK, limit))
+            if not out:
+                if inflater is not None:
+                    inflater.finish()
+                self._ended = True
+                return b""
+            if inflater is None:
+                return out
+            inflater.feed(out)
+
+    def read(self, size: "int | None" = -1) -> bytes:
+        """Up to `size` bytes of the decoded body, fewer only at its end; the
+        whole rest of it when `size` is negative or `None`."""
+        self._check_open()
+        parts = []
+        have = 0
+        while size is None or size < 0 or have < size:
+            wanted = CHUNK if size is None or size < 0 else size - have
+            chunk = self._leftover or self._next(wanted)
+            if not chunk:
+                break
+            if size is not None and 0 <= size < have + len(chunk):
+                self._leftover = chunk[size - have :]
+                chunk = chunk[: size - have]
+            else:
+                self._leftover = b""
+            parts.append(chunk)
+            have += len(chunk)
+        return b"".join(parts)
+
+    def iter_bytes(self, chunk_size: int = CHUNK) -> "_ty.Iterator[bytes]":
+        """The rest of the body in pieces of at most `chunk_size` bytes, each
+        as soon as it has arrived."""
+        self._check_open()
+        while True:
+            chunk = self._leftover or self._next(chunk_size)
+            if not chunk:
+                return
+            self._leftover = chunk[chunk_size:]
+            yield chunk[:chunk_size]
+
+    def close(self) -> None:
+        """Releases the connection. Harmless when called again. A body that was
+        not read to its end is abandoned, and its connection is closed."""
+        if not getattr(self, "_closed", True):
+            self._closed = True
+            self._source.close()
+
+    def buffered(self) -> "HTTPResponse":
+        """Reads the rest of the body into memory, closes the connection and
+        returns this response, which can then be read after it is closed. A
+        transport returns its response this way when `send()` is called with
+        `stream=False`."""
+        return self._buffered()
+
+    def _buffered(
+        self, limit: "int | None" = None, message: str = ""
+    ) -> "HTTPResponse":
+        """Reads the rest of the body into memory and closes the source, so the
+        response can be read after the connection has been given back. A
+        decoded body longer than `limit` bytes is `OSError(EFBIG)` with
+        `message`, and its connection is closed."""
+        try:
+            data = self.read(-1 if limit is None else limit + 1)
+            if limit is not None and len(data) > limit:
+                raise OSError(_errno.EFBIG, message or f"larger than {limit} bytes")
+            self._leftover = data
+            self._kept = True
+        finally:
+            self.close()
+        return self

pathapi/http/_settings.py

@@ -0,0 +1,12 @@
+"""The module-level setting of the HTTP client. Code reads it as an attribute
+of this module, so assigning it applies at once."""
+
+from __future__ import annotations
+
+MAX_REPLY_BYTES = 8 * 1024 * 1024
+"""Most bytes of a reply the library reads whole, after any content decoding:
+what `HTTPClient.request(stream=False)` returns, and a reply read for what it
+says (`read_limited()`, `json_body()`). A longer body raises `OSError(EFBIG)`
+naming the request and closes the connection. The body of a reply nobody looks
+at is read up to this many bytes and dropped (`drop_body()`). Raise it for a
+document that is legitimately larger. Read at the time of each use."""

pathapi/http/_status.py

@@ -0,0 +1,68 @@
+"""The status of a reply, and how much of its body the library reads."""
+
+from __future__ import annotations
+
+import json as _json
+
+from . import _settings
+from ._headers import peer_text
+from ._response import CHUNK, HTTPResponse
+
+
+class HTTPStatusError(Exception):
+    """A reply with an error status (4xx or 5xx). `response` is the reply;
+    `translate_errors()` turns the error into the exception `pathlib` raises."""
+
+    response: HTTPResponse
+
+    def __init__(self, response: HTTPResponse):
+        super().__init__(response.status)
+        self.response = response
+
+
+def raise_for_status(resp: HTTPResponse) -> None:
+    """Raises `HTTPStatusError` for a reply with status 400 or above."""
+    if resp.status >= 400:
+        raise HTTPStatusError(resp)
+
+
+def too_large(what: object, limit: int) -> str:
+    return (
+        f"The reply for {peer_text(what, 300)} is larger than {limit} bytes "
+        "(MAX_REPLY_BYTES)"
+    )
+
+
+def read_limited(resp: HTTPResponse, what: object, limit: "int | None" = None) -> bytes:
+    """The decoded body of `resp`, which must be at most `limit` bytes (default
+    `MAX_REPLY_BYTES`): `OSError(EFBIG)` naming `what` otherwise, with the
+    connection closed."""
+    limit = _settings.MAX_REPLY_BYTES if limit is None else limit
+    resp._buffered(limit, too_large(what, limit))
+    return resp.read()
+
+
+def drop_body(resp: HTTPResponse, limit: "int | None" = None) -> None:
+    """Gives up the body of a reply nobody looks at: it is read, up to `limit`
+    bytes (default `MAX_REPLY_BYTES`), so the connection can serve the next
+    request, and `resp` is closed (which closes the connection when the body
+    was longer). A failure while reading it is not the caller's concern: the
+    status was already used."""
+    remaining = _settings.MAX_REPLY_BYTES if limit is None else limit
+    try:
+        while remaining > 0:
+            chunk = resp.read(min(CHUNK, remaining))
+            if not chunk:
+                break
+            remaining -= len(chunk)
+    except (OSError, ValueError):
+        pass
+    finally:
+        resp.close()
+
+
+def json_body(resp: HTTPResponse, what: object, limit: "int | None" = None):
+    """The JSON document in the body of `resp` (at most `limit` bytes, default
+    `MAX_REPLY_BYTES`, else `OSError(EFBIG)` naming `what`); `ValueError` when
+    it is not one."""
+    return _json.loads(read_limited(resp, what, limit))

pathapi/http/_transports.py

@@ -0,0 +1,173 @@
+"""The HTTP transports that exist, which one is the default, and what a new one
+starts from."""
+
+from __future__ import annotations
+
+import os as _os
+import typing as _ty
+
+from ._body import prepare_body as _prepare_body
+from ._builtin import HTTPTransport
+from ._headers import (
+    check_body as _check_body,
+    check_headers as _check_headers,
+    check_method as _check_method,
+    check_timeout as _check_timeout,
+    check_url as _check_url,
+    has_header as _has_header,
+    wants_decoding as _wants_decoding,
+)
+
+TRANSPORT_GROUP = "pathapi.http_transports"
+"""The entry-point group of the HTTP transports. An entry's name is the
+transport's name (`requests`) and its value names a class that is built with
+no arguments."""
+
+TRANSPORT_ENV = "PATHAPI_HTTP_TRANSPORT"
+"""Names the transport an `HTTPClient` builds for itself: `builtin`, or the
+name of one that an installed distribution declares in `TRANSPORT_GROUP`
+(`requests`, from `pathapi-requests`). Read when that transport is first
+needed; unset means `builtin`."""
+
+_BUILTIN = "builtin"
+
+# Transport -> (the distribution that provides it, its import name, what to run
+# to get it). The name is known without the distribution, whose own entry in
+# `TRANSPORT_GROUP` is found first.
+_PROVIDED = {
+    "requests": (
+        "pathapi-requests",
+        "pathapi_requests",
+        "pip install pathapi-requests",
+    ),
+}
+
+# The methods whose request says `Content-Length: 0` when it has no body.
+_EXPECTING_BODY = frozenset({"PATCH", "POST", "PUT"})
+
+
+def _declared() -> "dict[str, _ty.Any]":
+    """The entry points of `TRANSPORT_GROUP` by name, the first declaration of
+    a name winning."""
+    import importlib.metadata as _metadata
+
+    try:
+        found = _metadata.entry_points(group=TRANSPORT_GROUP)
+    except TypeError:  # Python 3.9
+        found = _metadata.entry_points().get(TRANSPORT_GROUP, ())
+    declared: "dict[str, _ty.Any]" = {}
+    for entry in found:
+        declared.setdefault(entry.name, entry)
+    return declared
+
+
+def transport_names() -> "tuple[str, ...]":
+    """The names of the HTTP transports, sorted: `builtin`, every entry of
+    `TRANSPORT_GROUP`, and the transports that another distribution provides
+    whether or not it is installed (`requests`). Nothing is imported."""
+    return tuple(sorted({_BUILTIN, *_declared(), *_PROVIDED}))
+
+
+def get_transport(name: str) -> type:
+    """The class of the transport called `name` (case-sensitive), loading its
+    entry point. `builtin` is always `HTTPTransport`: no entry point replaces
+    it.
+
+    Raises `ImportError` naming what to install for a transport that another
+    distribution provides and that is not installed, and `ValueError` naming
+    the choices for any other name."""
+    if name == _BUILTIN:
+        return HTTPTransport
+    entry = _declared().get(name) if isinstance(name, str) else None
+    if entry is not None:
+        found = entry.load()
+        if not callable(found):
+            raise TypeError(f"the HTTP transport {name!r} is not a class")
+        return found
+    if isinstance(name, str) and name in _PROVIDED:
+        distribution, package, install = _PROVIDED[name]
+        raise ImportError(
+            f"the {name!r} HTTP transport is provided by the {distribution} "
+            f"distribution: {install}",
+            name=package,
+        )
+    raise ValueError(
+        f"{name!r} is not an HTTP transport "
+        f"(expected one of {', '.join(map(repr, transport_names()))})"
+    )
+
+
+def default_transport():
+    """A new transport of the kind `PATHAPI_HTTP_TRANSPORT` names. A value
+    that is empty, differently cased or unknown is `ValueError` naming the
+    choices; one that another distribution provides and that is not installed
+    is `ImportError` naming it."""
+    value = _os.environ.get(TRANSPORT_ENV)
+    kind = _BUILTIN if value is None else value
+    if kind == _BUILTIN:
+        return HTTPTransport()
+    try:
+        transport_class = get_transport(kind)
+    except ValueError:
+        raise ValueError(
+            f"{TRANSPORT_ENV}={kind!r} is not an HTTP transport "
+            f"(expected one of {', '.join(map(repr, transport_names()))})"
+        ) from None
+    return transport_class()
+
+
+class PreparedRequest(_ty.NamedTuple):
+    """A request that passed the checks of the transport contract.
+
+    `headers` hold text values, with the `Content-Length` of a body whose
+    length is known (`0` for a `PATCH`, `POST` or `PUT` without a body) and an
+    `Accept-Encoding` of `gzip, deflate` unless the caller named one. `body` is
+    `None`, `bytes`, an object that sends exactly the `Content-Length` bytes
+    when it is read or iterated, or an iterable of `bytes` to send chunked.
+    `timeout` is `(connect, read)`. `decode` says whether the response decodes
+    a `gzip` or `deflate` body: build it with `HTTPResponse(...,
+    decode=decode)`."""
+
+    method: str
+    url: str
+    headers: "dict[str, str]"
+    body: _ty.Any
+    timeout: "tuple[float | None, float | None]"
+    decode: bool
+
+
+def prepare_request(
+    method: str,
+    url: str,
+    *,
+    headers: "_ty.Mapping[str, str] | None",
+    body: "bytes | _ty.BinaryIO | _ty.Iterable[bytes] | None" = None,
+    timeout: "float | tuple | None",
+) -> PreparedRequest:
+    """The arguments of `send()` as the contract reads them.
+
+    `method` is upper-cased. Raises `ValueError`, before anything is sent, for
+    a method or header name that is not an HTTP token, a header value or URL
+    with a control character, a URL with credentials or another scheme, a
+    `Content-Length` that is not a count of bytes or exceeds the body, and a
+    `timeout` that is not a positive number, `None` or a pair of them;
+    `TypeError` for a `str` body. None of the messages quotes the URL or a
+    header value. The fragment is dropped from `url`."""
+    method = _check_method(method)
+    _check_url(url)
+    _check_body(body)
+    timeout = _check_timeout(timeout)
+    pairs = _check_headers(headers)
+    sent = dict(pairs)
+    body, length = _prepare_body(body, pairs)
+    if length is not None:
+        sent["Content-Length"] = str(length)
+    elif not _has_header(sent, "Content-Length"):
+        if isinstance(body, (bytes, bytearray, memoryview)):
+            sent["Content-Length"] = str(memoryview(body).nbytes)
+        elif body is None and method in _EXPECTING_BODY:
+            sent["Content-Length"] = "0"
+    decode = _wants_decoding(sent.items())
+    if not _has_header(sent, "Accept-Encoding"):
+        sent["Accept-Encoding"] = "gzip, deflate"
+    return PreparedRequest(method, url.split("#", 1)[0], sent, body, timeout, decode)

pathapi/http/testing/__init__.py

@@ -0,0 +1,80 @@
+"""Testing support: the suite that holds every HTTP transport to the same
+promises.
+
+This module is testing support, not part of the stable API: it is importable
+and documented, but its contents (the test names, the fixtures, the helper
+signatures, `MODULES` and `HELPERS`) may change in any release. `pathapi.http`
+never imports it.
+
+`HTTPClient` and the transport contract are tested over a transport chosen by
+name, against loopback servers scripted at the byte level. A distribution that
+provides a transport (see `pathapi.http.get_transport`) runs the same
+tests over its own, with its distribution installed::
+
+    python -m pathapi.http.testing --http-transports requests -q
+
+Everything after `python -m pathapi.http.testing` is passed to pytest,
+which this module needs. `--http-transports` takes names that
+`pathapi.http.transport_names()` lists, separated by commas; it defaults to
+`builtin`. Every request goes to the loopback interface: the suite fails a test
+that tries to reach any other host. The TLS tests need `cryptography` and skip
+without it. This module imports nothing but the standard library: importing it
+loads `pathapi.http` (and so `ssl`) but not `pytest`, which `run()` imports.
+"""
+
+from __future__ import annotations
+
+import pathlib as _pathlib
+import typing as _ty
+
+__all__ = ["HELPERS", "MODULES", "run", "suite_directory"]
+
+MODULES = (
+    "test_client_default.py",
+    "test_client_errors.py",
+    "test_client_peer_text.py",
+    "test_client_redirect_origin.py",
+    "test_client_redirects.py",
+    "test_client_reply_limits.py",
+    "test_client_write_redirects.py",
+    "test_transport_body.py",
+    "test_transport_send.py",
+)
+"""The test modules that run once per transport, by file name."""
+
+HELPERS = (
+    "conftest.py",
+    "transport_fakes.py",
+    "transport_guard.py",
+    "transport_wire.py",
+)
+"""The files beside them that they import: the fixtures and the loopback servers."""
+
+
+def suite_directory() -> _pathlib.Path:
+    """The directory that holds `MODULES` and `HELPERS`: the copy installed
+    with the package, or `tests/transport/` of a source checkout. Raises
+    `FileNotFoundError` when neither is there."""
+    here = _pathlib.Path(__file__).resolve().parent
+    for candidate in (here / "suite", here.parents[3] / "tests" / "transport"):
+        if all((candidate / name).is_file() for name in (*MODULES, *HELPERS)):
+            return candidate
+    raise FileNotFoundError("the transport suite is not installed with pathapi")
+
+
+def run(arguments: "_ty.Sequence[str] | None" = None) -> int:
+    """Runs `MODULES` with pytest and `arguments` (pytest options, such as
+    `--http-transports requests`) and returns the exit status. Raises
+    `ImportError` when pytest is not installed."""
+    try:
+        import pytest
+    except ImportError as error:
+        raise ImportError(
+            "the transport suite needs pytest: pip install pytest", name="pytest"
+        ) from error
+    directory = suite_directory()
+    files = [str(directory / name) for name in MODULES]
+    # The files go first: with an option and its value ahead of them, pytest
+    # 8 reads the value as the first path and loads no `conftest.py`, so the
+    # option `--http-transports` is not known.
+    return int(pytest.main([*files, *(arguments or ())]))

pathapi/http/testing/__main__.py

@@ -0,0 +1,10 @@
+"""`python -m pathapi.http.testing [pytest options]` runs the transport suite."""
+
+from __future__ import annotations
+
+import sys
+
+from . import run
+
+if __name__ == "__main__":
+    raise SystemExit(run(sys.argv[1:]))