syshttp 0.1.0__py3-none-any.whl

syshttp/client/_pool.py

@@ -0,0 +1,170 @@
+"""The idle connections of a transport: one per thread and server."""
+
+from __future__ import annotations
+
+import collections as _collections
+import itertools as _itertools
+import socket as _socket
+import threading as _threading
+import typing as _ty
+import weakref as _weakref
+
+MAX_IDLE_PER_THREAD = 8
+"""Most idle connections one thread keeps open; opening another closes the one
+used longest ago."""
+
+
+class Connection:
+    """An open `http.client.HTTPConnection` and what the pool knows of it."""
+
+    __slots__ = ("http", "key", "scope", "released_at", "generation", "__weakref__")
+
+    def __init__(self, http, key: tuple, scope: int, generation: int):
+        self.http = http
+        self.key = key
+        self.scope = scope
+        self.generation = generation
+        self.released_at = 0.0
+
+    def close(self, interrupt: bool = False) -> None:
+        """Closes the socket. `interrupt` also shuts it down first, which wakes
+        a thread blocked on it."""
+        http = self.http
+        sock = getattr(http, "sock", None)
+        try:
+            if interrupt and sock is not None:
+                try:
+                    sock.shutdown(_socket.SHUT_RDWR)
+                except OSError:
+                    pass
+        finally:
+            try:
+                http.close()
+            except Exception:
+                pass
+
+
+_SCOPE_IDS = _itertools.count(1)
+_THREAD = _threading.local()
+_LIVE_SCOPES: "set[int]" = set()
+_POOLS: "_weakref.WeakSet[Pool]" = _weakref.WeakSet()
+
+
+def _drop_scope(scope: int) -> None:
+    _LIVE_SCOPES.discard(scope)
+    for pool in list(_POOLS):
+        pool.drop_scope(scope)
+
+
+class _ThreadScope:
+    """Held in the thread's own storage, so it is released when the thread ends
+    and takes the thread's idle connections with it."""
+
+    def __init__(self):
+        self.id = next(_SCOPE_IDS)
+        _LIVE_SCOPES.add(self.id)
+        _weakref.finalize(self, _drop_scope, self.id).atexit = False
+
+
+def scope_id() -> int:
+    """The calling thread's scope. Not the thread identifier, which the
+    operating system reuses for a later thread."""
+    scope = getattr(_THREAD, "scope", None)
+    if scope is None:
+        scope = _THREAD.scope = _ThreadScope()
+    return scope.id
+
+
+class Pool:
+    """Idle connections by (thread scope, server), and every connection a
+    transport has open, so `close()` reaches the ones in use too. A connection
+    is taken out of the pool for the length of a response and given back once
+    the body has been read."""
+
+    def __init__(self):
+        self._lock = _threading.Lock()
+        self._idle: "dict[int, _collections.OrderedDict[tuple, Connection]]" = {}
+        self._open: "set[Connection]" = set()
+        self._generation = 0
+        _POOLS.add(self)
+
+    @property
+    def generation(self) -> int:
+        return self._generation
+
+    def take(self, key: tuple, scope: int) -> "Connection | None":
+        """The idle connection of the calling thread to `key`, removed from the
+        pool, or `None`."""
+        with self._lock:
+            idle = self._idle.get(scope)
+            return idle.pop(key, None) if idle else None
+
+    def has_idle(self, key: tuple, scope: int) -> bool:
+        """Whether the calling thread has an idle connection to `key`."""
+        with self._lock:
+            return key in self._idle.get(scope, ())
+
+    def register(self, connection: Connection) -> None:
+        with self._lock:
+            self._open.add(connection)
+
+    def give_back(self, connection: Connection, now: float) -> None:
+        """Keeps `connection` idle for the next request of its thread, or
+        closes it when its thread has ended, the transport was closed since it
+        was opened, or the thread already holds one to the same server."""
+        connection.released_at = now
+        evicted = []
+        with self._lock:
+            keep = connection.scope in _LIVE_SCOPES and connection.generation == self._generation
+            if keep:
+                idle = self._idle.setdefault(connection.scope, _collections.OrderedDict())
+                if connection.key in idle:
+                    keep = False
+                else:
+                    idle[connection.key] = connection
+                    while len(idle) > MAX_IDLE_PER_THREAD:
+                        evicted.append(idle.popitem(last=False)[1])
+            if not keep:
+                self._open.discard(connection)
+                evicted.append(connection)
+            for old in evicted:
+                self._open.discard(old)
+        for old in evicted:
+            old.close()
+
+    def discard(self, connection: Connection) -> None:
+        """Closes a connection that is not to be used again."""
+        with self._lock:
+            self._open.discard(connection)
+        connection.close()
+
+    def drop_scope(self, scope: int) -> None:
+        with self._lock:
+            idle: "dict[tuple, Connection]" = self._idle.pop(scope, {})
+            for connection in idle.values():
+                self._open.discard(connection)
+        for connection in list(idle.values()):
+            connection.close()
+
+    def close(self) -> None:
+        """Closes every connection, idle or in use. Harmless when called again;
+        a later request connects afresh."""
+        with self._lock:
+            self._generation += 1
+            doomed = list(self._open)
+            self._open.clear()
+            self._idle.clear()
+        for connection in doomed:
+            connection.close(interrupt=True)
+
+
+def close_pool(pool: Pool) -> None:
+    pool.close()
+
+
+def finalize_with(owner: object, pool: Pool) -> "_ty.Any":
+    """Closes `pool` when `owner` is collected, so a transport that was never
+    closed does not leave sockets for the interpreter to report."""
+    handle = _weakref.finalize(owner, close_pool, pool)
+    handle.atexit = False  # type: ignore[misc]
+    return handle

syshttp/client/_prepare.py

@@ -0,0 +1,77 @@
+"""What a transport starts from: a request that passed the checks every transport
+holds itself to."""
+
+from __future__ import annotations
+
+import typing as _ty
+
+from ._body import prepare_body as _prepare_body
+from ._headers import (
+    check_body as _check_body,
+    check_headers as _check_headers,
+    check_method as _check_method,
+    check_timeout as _check_timeout,
+    check_url as _check_url,
+    has_header as _has_header,
+    wants_decoding as _wants_decoding,
+)
+
+# The methods whose request says `Content-Length: 0` when it has no body.
+_EXPECTING_BODY = frozenset({"PATCH", "POST", "PUT"})
+
+
+class PreparedRequest(_ty.NamedTuple):
+    """A request that passed the checks of the transport contract.
+
+    `headers` hold text values, with the `Content-Length` of a body whose
+    length is known (`0` for a `PATCH`, `POST` or `PUT` without a body) and an
+    `Accept-Encoding` of `gzip, deflate` unless the caller named one. `body` is
+    `None`, `bytes`, an object that sends exactly the `Content-Length` bytes
+    when it is read or iterated, or an iterable of `bytes` to send chunked.
+    `timeout` is `(connect, read)`. `decode` says whether the response decodes
+    a `gzip` or `deflate` body: build it with `HTTPResponse(...,
+    decode=decode)`."""
+
+    method: str
+    url: str
+    headers: "dict[str, str]"
+    body: _ty.Any
+    timeout: "tuple[float | None, float | None]"
+    decode: bool
+
+
+def prepare_request(
+    method: str,
+    url: str,
+    *,
+    headers: "_ty.Mapping[str, str] | None",
+    body: "bytes | _ty.BinaryIO | _ty.Iterable[bytes] | None" = None,
+    timeout: "float | tuple | None",
+) -> PreparedRequest:
+    """The arguments of `send()` as the contract reads them.
+
+    `method` is upper-cased. Raises `ValueError`, before anything is sent, for
+    a method or header name that is not an HTTP token, a header value or URL
+    with a control character, a URL with credentials or another scheme, a
+    `Content-Length` that is not a count of bytes or exceeds the body, and a
+    `timeout` that is not a positive number, `None` or a pair of them;
+    `TypeError` for a `str` body. None of the messages quotes the URL or a
+    header value. The fragment is dropped from `url`."""
+    method = _check_method(method)
+    _check_url(url)
+    _check_body(body)
+    timeout = _check_timeout(timeout)
+    pairs = _check_headers(headers)
+    sent = dict(pairs)
+    body, length = _prepare_body(body, pairs)
+    if length is not None:
+        sent["Content-Length"] = str(length)
+    elif not _has_header(sent, "Content-Length"):
+        if isinstance(body, (bytes, bytearray, memoryview)):
+            sent["Content-Length"] = str(memoryview(body).nbytes)
+        elif body is None and method in _EXPECTING_BODY:
+            sent["Content-Length"] = "0"
+    decode = _wants_decoding(sent.items())
+    if not _has_header(sent, "Accept-Encoding"):
+        sent["Accept-Encoding"] = "gzip, deflate"
+    return PreparedRequest(method, url.split("#", 1)[0], sent, body, timeout, decode)

syshttp/client/_response.py

@@ -0,0 +1,301 @@
+"""The response every transport returns, with the decoding they share."""
+
+from __future__ import annotations
+
+import codecs as _codecs
+import email.message as _message
+import errno as _errno
+import json as _json
+import typing as _ty
+import zlib as _zlib
+
+from ..exceptions import ResponseError
+from . import _settings
+from ._headers import Headers, peer_text, shown
+
+CHUNK = 64 * 1024
+"""Most bytes read from a transport's body in one call."""
+
+_STEP = 256 * 1024
+"""Most decoded bytes one decompression step produces. The compressed bytes
+that are left wait for the next step, so a body that expands a thousandfold
+is never held whole because its header said it was small."""
+
+
+def too_large(what: object, limit: int) -> str:
+    """The message of the `OSError(EFBIG)` for a reply longer than `limit` bytes."""
+    return f"The reply for {peer_text(what, 300)} is larger than {limit} bytes (MAX_REPLY_BYTES)"
+
+
+class _Memory:
+    """A body that is already in memory."""
+
+    def __init__(self, data):
+        self._data = memoryview(bytes(data))
+        self._at = 0
+
+    def read(self, amount: int) -> bytes:
+        chunk = bytes(self._data[self._at : self._at + amount])
+        self._at += len(chunk)
+        return chunk
+
+    def close(self) -> None:
+        pass
+
+
+class _Inflater:
+    """Decompresses one `gzip` or `deflate` body in bounded steps. `deflate` is
+    read as zlib-wrapped data, or as raw data when the zlib header is absent,
+    which is what servers send under that name; concatenated gzip members are
+    one body."""
+
+    def __init__(self, gzip: bool):
+        self._gzip = gzip
+        self._object: _ty.Any = None
+        self._input = b""
+        self._members = 0
+
+    @property
+    def has_input(self) -> bool:
+        return bool(self._input)
+
+    def feed(self, data: bytes) -> None:
+        self._input += data
+
+    def _new(self, data: bytes):
+        if self._gzip:
+            return _zlib.decompressobj(16 + _zlib.MAX_WBITS)
+        wrapped = len(data) < 2 or (data[0] & 0x0F == 8 and ((data[0] << 8) | data[1]) % 31 == 0)
+        return _zlib.decompressobj(_zlib.MAX_WBITS if wrapped else -_zlib.MAX_WBITS)
+
+    def step(self) -> bytes:
+        data = self._input
+        if self._object is None:
+            self._object = self._new(data)
+        try:
+            out = self._object.decompress(data, _STEP)
+        except _zlib.error:
+            if self._members:
+                # Bytes after the last complete member are not a body.
+                self._input, self._object = b"", None
+                return b""
+            kind = "gzip" if self._gzip else "deflate"
+            raise OSError(_errno.EIO, f"the response body is not valid {kind} data") from None
+        if self._object.eof:
+            self._members += 1
+            self._input = self._object.unused_data
+            self._object = None
+        else:
+            self._input = self._object.unconsumed_tail
+            if not out and len(self._input) >= len(data):
+                self._input = b""
+        return out
+
+    def finish(self) -> None:
+        if self._object is not None:
+            raise OSError(_errno.EIO, "the compressed response body ended early")
+
+
+def _inflater_for(headers: Headers) -> "_Inflater | None":
+    names = [
+        name.strip().lower()
+        for name in headers.get("Content-Encoding", "").split(",")
+        if name.strip().lower() not in ("", "identity")
+    ]
+    if len(names) != 1:
+        return None
+    if names[0] in ("gzip", "x-gzip"):
+        return _Inflater(gzip=True)
+    if names[0] == "deflate":
+        return _Inflater(gzip=False)
+    return None
+
+
+class HTTPResponse:
+    """One response: `status`, `reason`, `headers` (a read-only mapping that
+    ignores the case of a name), `url` (the URL the request went to) and the
+    body, read with `read()` or `iter_bytes()`. A context manager that closes.
+
+    The body is decoded when the response says it is `gzip` or `deflate`
+    encoded, unless the request set its own `Accept-Encoding`; `headers` are
+    the headers as received, so `Content-Length` counts the encoded bytes. A
+    body that ends before the length the server stated is `OSError(EIO)`, never
+    a shorter body that reads as complete.
+
+    A transport builds one with `body` as `bytes`, or as an object with
+    `read(amount) -> bytes` (empty at the end) and `close()`, and
+    `decode=False` for a body it already decoded or that was not asked for
+    in a coding."""
+
+    status: int
+    reason: str
+    headers: Headers
+    url: str
+
+    def __init__(
+        self,
+        status: int,
+        reason: str,
+        headers: "_ty.Mapping[str, str] | _ty.Iterable[tuple[str, str]]",
+        url: str,
+        body: "bytes | _ty.Any" = b"",
+        *,
+        decode: bool = True,
+    ):
+        self._closed = False
+        self._ended = False
+        self.status = int(status)
+        self.reason = reason or ""
+        self.headers = headers if isinstance(headers, Headers) else Headers(headers)
+        self.url = url
+        self._kept = False
+        self._content: "bytes | None" = None
+        if isinstance(body, (bytes, bytearray, memoryview)):
+            body = _Memory(body)
+            self._kept = True  # in memory: readable after close
+        self._source = body
+        self._inflater = _inflater_for(self.headers) if decode else None
+        self._leftover = b""
+
+    def __repr__(self) -> str:
+        return f"HTTPResponse({self.status}, {self.reason!r})"
+
+    def __enter__(self) -> "HTTPResponse":
+        return self
+
+    def __exit__(self, *exc_info) -> None:
+        self.close()
+
+    def __del__(self):
+        try:
+            self.close()
+        except Exception:
+            pass
+
+    def _check_open(self) -> None:
+        if self._closed and not self._kept:
+            raise ValueError("read of a closed response")
+
+    def _next(self, limit: int = CHUNK) -> bytes:
+        """The next decoded bytes of the body; empty at its end. Without a
+        coding, no more than `limit` bytes are asked of the transport."""
+        if self._ended:
+            return b""
+        self._check_open()
+        inflater = self._inflater
+        while True:
+            if inflater is not None and inflater.has_input:
+                out = inflater.step()
+                if out:
+                    return out
+                continue
+            out = self._source.read(CHUNK if inflater else min(CHUNK, limit))
+            if not out:
+                if inflater is not None:
+                    inflater.finish()
+                self._ended = True
+                return b""
+            if inflater is None:
+                return out
+            inflater.feed(out)
+
+    def read(self, size: "int | None" = -1) -> bytes:
+        """Up to `size` bytes of the decoded body, fewer only at its end; the
+        whole rest of it when `size` is negative or `None`."""
+        self._check_open()
+        parts = []
+        have = 0
+        while size is None or size < 0 or have < size:
+            wanted = CHUNK if size is None or size < 0 else size - have
+            chunk = self._leftover or self._next(wanted)
+            if not chunk:
+                break
+            if size is not None and 0 <= size < have + len(chunk):
+                self._leftover = chunk[size - have :]
+                chunk = chunk[: size - have]
+            else:
+                self._leftover = b""
+            parts.append(chunk)
+            have += len(chunk)
+        return b"".join(parts)
+
+    def iter_bytes(self, chunk_size: int = CHUNK) -> "_ty.Iterator[bytes]":
+        """The rest of the body in pieces of at most `chunk_size` bytes, each
+        as soon as it has arrived."""
+        self._check_open()
+        while True:
+            chunk = self._leftover or self._next(chunk_size)
+            if not chunk:
+                return
+            self._leftover = chunk[chunk_size:]
+            yield chunk[:chunk_size]
+
+    @property
+    def content(self) -> bytes:
+        """The rest of the decoded body, read on first use and kept, so that
+        `content`, `text()` and `json()` agree however often they are used. A
+        body longer than `MAX_REPLY_BYTES` is `OSError(EFBIG)`. Bytes already
+        taken with `read()` or `iter_bytes()` are not in it."""
+        if self._content is None:
+            limit = _settings.MAX_REPLY_BYTES
+            self._buffered(limit, too_large(shown(self.url), limit))
+            self._content = self.read()
+        return self._content
+
+    def text(self, encoding: "str | None" = None, errors: str = "replace") -> str:
+        """`content` decoded with `encoding`, else the `charset` of the
+        `Content-Type` header, else UTF-8; characters that do not decode are
+        replaced unless `errors` says otherwise."""
+        charset = encoding or self._charset() or "utf-8"
+        try:
+            _codecs.lookup(charset)
+        except LookupError:
+            charset = "utf-8"
+        return self.content.decode(charset, errors)
+
+    def json(self) -> _ty.Any:
+        """`content` parsed as JSON; `None` for an empty body. A body that is
+        not JSON is `ResponseError` (a `ValueError`) naming the status."""
+        data = self.content
+        if not data.strip():
+            return None
+        try:
+            return _json.loads(data)
+        except ValueError as error:
+            raise ResponseError(f"response was not valid JSON (HTTP {self.status})") from error
+
+    def _charset(self) -> "str | None":
+        message = _message.Message()
+        message["Content-Type"] = self.headers.get("Content-Type", "")
+        return message.get_content_charset()
+
+    def close(self) -> None:
+        """Releases the connection. Harmless when called again. A body that was
+        not read to its end is abandoned, and its connection is closed."""
+        if not getattr(self, "_closed", True):
+            self._closed = True
+            self._source.close()
+
+    def buffered(self, limit: "int | None" = None) -> "HTTPResponse":
+        """Reads the rest of the body into memory, closes the connection and
+        returns this response, which can then be read after it is closed. A
+        transport returns its response this way when `send()` is called with
+        `stream=False`. A body that decodes to more than `limit` bytes (default
+        `MAX_REPLY_BYTES`) is `OSError(EFBIG)`, and its connection is closed."""
+        limit = _settings.MAX_REPLY_BYTES if limit is None else limit
+        return self._buffered(limit, too_large(shown(self.url), limit))
+
+    def _buffered(self, limit: "int | None" = None, message: str = "") -> "HTTPResponse":
+        """Reads the rest of the body into memory and closes the source, so the
+        response can be read after the connection has been given back. A
+        decoded body longer than `limit` bytes is `OSError(EFBIG)` with
+        `message`, and its connection is closed."""
+        try:
+            data = self.read(-1 if limit is None else limit + 1)
+            if limit is not None and len(data) > limit:
+                raise OSError(_errno.EFBIG, message or f"larger than {limit} bytes")
+            self._leftover = data
+            self._kept = True
+        finally:
+            self.close()
+        return self

syshttp/client/_route.py

@@ -0,0 +1,75 @@
+"""The route of a request: what the proxy map answers, narrowed to the entries
+the transport speaks and to the first one that is reachable."""
+
+from __future__ import annotations
+
+import base64 as _base64
+import typing as _ty
+
+import proxylib
+
+DEFAULT_PORTS = {"http": 80, "https": 443, "prefix+http": 80, "prefix+https": 443}
+"""The proxy schemes the transport speaks, and their ports."""
+
+PROBE_TIMEOUT = 5.0
+"""Most seconds spent finding out whether one proxy of an answer accepts a
+connection."""
+
+Entry = _ty.Optional["proxylib.Proxy"]
+
+
+def entries(proxymap: _ty.Any, url: str) -> "tuple[Entry, ...]":
+    """The entries of `proxymap`'s answer for `url` that the transport can use,
+    in order (`None` is a direct connection); `(None,)` when the map has no
+    opinion. `proxylib.UnsupportedProxyError` when it names only proxies the
+    transport cannot speak (SOCKS)."""
+    try:
+        answer = proxylib.lookup(proxymap, url)
+    except KeyError:
+        return (None,)
+    usable: "list[Entry]" = []
+    refused: "list[tuple[proxylib.Proxy, str]]" = []
+    for entry in answer:
+        why = None if entry is None else _refusal(entry, url)
+        if why is None:
+            usable.append(entry)
+        else:
+            refused.append((_ty.cast("proxylib.Proxy", entry), why))
+    if usable:
+        return tuple(usable)
+    raise proxylib.UnsupportedProxyError("syshttp.client", refused)
+
+
+def _refusal(entry: "proxylib.Proxy", url: str) -> "str | None":
+    """Why the transport does not send `url` through `entry`, or `None`."""
+    if entry.scheme not in DEFAULT_PORTS:
+        return "only http and https proxies and URL-prefix gateways are spoken"
+    if entry.scheme == "prefix+http" and url[:6].lower() == "https:":
+        return "an https origin's request would cross a plain-http gateway in clear text"
+    return None
+
+
+def reachable(candidates: "tuple[Entry, ...]", url: str, connect_timeout: "float | None") -> Entry:
+    """The first of `candidates` that accepts a TCP connection (a direct entry
+    always does). `proxylib.NoReachableProxyError` when none does."""
+    timeout = PROBE_TIMEOUT if connect_timeout is None else min(connect_timeout, PROBE_TIMEOUT)
+    return proxylib.lookup(proxylib.Probed(proxylib.SimpleProxyMap(candidates), timeout=timeout), url)[0]
+
+
+def port(proxy: "proxylib.Proxy") -> int:
+    """The port a connection to `proxy` is made to."""
+    return proxy.port or DEFAULT_PORTS[proxy.scheme]
+
+
+def credentials(proxy: "proxylib.Proxy") -> "list[tuple[str, str]]":
+    """The headers a request to `proxy` carries for it: its own headers, and
+    its userinfo as Basic `Proxy-Authorization` unless one is among them. A
+    value that is not ASCII is `ValueError`, without the value."""
+    found = list(proxy.headers.items())
+    if proxy.username is not None and not any(name.lower() == "proxy-authorization" for name, _ in found):
+        token = f"{proxy.username}:{proxy.password or ''}".encode("utf-8")
+        found.append(("Proxy-Authorization", "Basic " + _base64.b64encode(token).decode("ascii")))
+    for name, value in found:
+        if not value.isascii():
+            raise ValueError(f"the value of the proxy header {name} is not ASCII")
+    return found

syshttp/client/_settings.py

@@ -0,0 +1,12 @@
+"""The module-level setting of the HTTP client. Code reads it as an attribute
+of this module, so assigning it applies at once."""
+
+from __future__ import annotations
+
+MAX_REPLY_BYTES = 8 * 1024 * 1024
+"""Most bytes of a reply the library reads whole, after any content decoding:
+what `HTTPClient.request(stream=False)` returns, and a reply read for what it
+says (`read_limited()`, `json_body()`). A longer body raises `OSError(EFBIG)`
+naming the request and closes the connection. The body of a reply nobody looks
+at is read up to this many bytes and dropped (`drop_body()`). Raise it for a
+document that is legitimately larger. Read at the time of each use."""