syshttp 0.1.0__py3-none-any.whl

syshttp/autopatch.py

@@ -0,0 +1,382 @@
+"""Patch HTTP client libraries as they are imported, so they route and verify
+the way syshttp does: routing by :func:`syshttp.proxy.proxy_map` through
+proxylib's client integrations (``proxylib.patch``), and TLS by the system's
+context (:func:`syshttp.tls.os_ssl_context`).
+
+``SYSHTTP_IMPORTPATCH`` chooses the libraries -- read under the prefixes as
+every setting is (``SYSHTTP_PREFIX=AGENTS_``: ``AGENTS_SYSHTTP_IMPORTPATCH``): ``1``/``true``/``yes``/``on`` for all of
+:data:`TARGETS`, ``0``/``false``/``no``/``off``, empty or unset for none, else
+their names separated by ``:`` (``requests:httpx``; a module name such as
+``urllib.request`` works too). It is read when the library is imported:
+importpatch (a dependency) calls :func:`importpatch` for each target, through
+this distribution's ``importpatch`` entry points. :func:`patch` applies the
+same patches from code, whatever the setting says.
+
+What the caller chose always wins: a proxy given to the client, a context, a
+CA bundle or ``verify=False``.
+"""
+
+from __future__ import annotations
+
+import functools
+import importlib
+import logging
+import ssl
+import sys
+import threading
+import time
+from types import ModuleType
+from typing import Any, Callable, Dict, FrozenSet, Iterable, List, Optional, Tuple
+
+# Neither proxylib nor syshttp.proxy is imported here: proxylib's own import
+# of urllib.request runs this module's hook while proxylib is half built.
+from .env import HTTPEnv
+
+__all__ = ["SETTING", "TARGETS", "enabled", "importpatch", "patch", "patched", "setting_name", "unpatch"]
+
+#: The setting, read under every prefix as any other is (:func:`setting_name`).
+SETTING = "SYSHTTP_IMPORTPATCH"
+#: The libraries that can be patched: name -> the module whose import patches it.
+TARGETS: Dict[str, str] = {
+    "requests": "requests",
+    "urllib": "urllib.request",
+    "httpx": "httpx",
+    "aiohttp": "aiohttp",
+    "pip": "pip._vendor.requests",
+}
+
+_log = logging.getLogger(__name__)
+_TRUE = ("1", "true", "yes", "on")
+_FALSE = ("", "0", "false", "no", "off")
+_BY_MODULE = {module: name for name, module in TARGETS.items()}
+_lock = threading.RLock()
+# name -> the functions that undo its patch, in the order applied.
+_undo: Dict[str, List[Callable[[], None]]] = {}
+
+
+def setting_name(*, env: Optional[HTTPEnv] = None) -> str:
+    """The variable that chooses the libraries: ``<PREFIX>SYSHTTP_IMPORTPATCH``
+    for the first prefix that sets it (``AGENTS_SYSHTTP_IMPORTPATCH`` under
+    ``SYSHTTP_PREFIX=AGENTS_``), else the name under the first prefix."""
+    env = HTTPEnv() if env is None else env
+    found = env.find(SETTING)
+    return found[0] if found else env.name(SETTING)
+
+
+def enabled(*, env: Optional[HTTPEnv] = None) -> FrozenSet[str]:
+    """The names of :data:`TARGETS` that :func:`setting_name` enables, ``:``
+    separated. A name that is not a target is logged at WARNING and skipped."""
+    env = HTTPEnv() if env is None else env
+    name = setting_name(env=env)
+    raw = env.get(SETTING) or ""
+    if raw.lower() in _FALSE:
+        return frozenset()
+    if raw.lower() in _TRUE:
+        return frozenset(TARGETS)
+    names = set()
+    for part in raw.split(":"):
+        part = part.strip()
+        if not part:
+            continue
+        target = _BY_MODULE.get(part, part)
+        if target in TARGETS:
+            names.add(target)
+        else:
+            _log.warning("%s names %r, which is not one of %s", name, part, ", ".join(TARGETS))
+    return frozenset(names)
+
+
+def importpatch(name: str, module: ModuleType) -> None:
+    """importpatch's entry point: patch ``module``, just imported as ``name``,
+    when :func:`setting_name` enables it."""
+    _flush()
+    target = _BY_MODULE.get(name)
+    if target is not None and target in enabled():
+        _apply(target, module)
+
+
+def patch(targets: Optional[Iterable[str]] = None) -> Tuple[str, ...]:
+    """Patch ``targets`` now (default: every target whose library is
+    installed), importing each library. A name that is not a target is a
+    ``ValueError``; a named library that is not installed raises its
+    ``ImportError``. Returns the names patched; patching twice does nothing."""
+    names = list(TARGETS) if targets is None else [_BY_MODULE.get(t, t) for t in targets]
+    for name in names:
+        if name not in TARGETS:
+            raise ValueError("%r is not one of %s" % (name, ", ".join(TARGETS)))
+    _flush()
+    done = []
+    for name in names:
+        try:
+            module = importlib.import_module(TARGETS[name])
+        except ImportError:
+            if targets is not None:
+                raise
+            continue
+        _apply(name, module)
+        done.append(name)
+    return tuple(done)
+
+
+def unpatch() -> None:
+    """Undo every patch, the last applied first. A client built while patched
+    keeps what it was given; proxylib's integrations stay installed, back on
+    proxylib's default map."""
+    with _lock:
+        for name in reversed(list(_undo)):
+            for step in reversed(_undo.pop(name)):
+                step()
+
+
+def patched() -> Tuple[str, ...]:
+    """The names patched now, in the order applied."""
+    with _lock:
+        return tuple(_undo)
+
+
+def _apply(name: str, module: ModuleType) -> None:
+    with _lock:
+        if name in _undo:
+            return
+        steps: List[Callable[[], None]] = []
+        if _initializing("proxylib") or _initializing("syshttp.proxy"):
+            steps.append(_defer(name, module))
+        else:
+            # Before the TLS patch: syshttp's wrappers then run first and
+            # hand proxylib's transports the system context.
+            steps.extend(_route(name))
+        tls = _TLS[name]
+        if tls is not None:
+            steps.append(tls(module))
+        _undo[name] = steps
+
+
+def _initializing(name: str) -> bool:
+    """Whether module ``name`` is being imported right now (half built)."""
+    module = sys.modules.get(name)
+    return module is not None and bool(getattr(getattr(module, "__spec__", None), "_initializing", False))
+
+
+def _context(certificate: bool = False) -> ssl.SSLContext:
+    """The system's context; a fresh one offering no store certificate when
+    the caller loads a certificate of its own into it."""
+    from . import tls as _tls
+
+    if certificate:
+        return _tls.new_os_context(client_certificates=False)
+    return _tls.os_ssl_context()
+
+
+# ---- routing -----------------------------------------------------------------
+
+
+def _route(name: str) -> List[Callable[[], None]]:
+    """``proxylib.patch`` with syshttp's map for target ``name``; nothing
+    when this proxylib has no such target."""
+    import proxylib
+
+    from . import proxy as _proxy
+
+    if name not in proxylib.patching.targets():
+        _log.debug("proxylib %s has no %r patch target: %s is not routed", proxylib.__version__, name, name)
+        return []
+    proxymap = _URLErrorMap(_proxy.proxy_map()) if name == "urllib" else _proxy.proxy_map()
+    return [proxylib.patch(proxymap, targets=[name]).undo]
+
+
+# Targets whose routing waits for proxylib to finish importing -> their undo
+# steps once routed (None until then).
+_pending: Dict[str, Optional[List[Callable[[], None]]]] = {}
+
+
+def _defer(name: str, module: ModuleType) -> Callable[[], None]:
+    """Route ``name`` once proxylib is importable: at the next patch, or for
+    urllib (the only client proxylib imports itself) at the first
+    ``urlopen``, the call proxylib's urllib integration serves."""
+    _pending[name] = None
+    undo_opener: Callable[[], None] = lambda: None
+    if name == "urllib":
+        original = getattr(module, "urlopen")
+
+        @functools.wraps(original)
+        def urlopen(*args: Any, **kwargs: Any) -> Any:
+            _flush()
+            return original(*args, **kwargs)
+
+        setattr(module, "urlopen", urlopen)
+        undo_opener = lambda: setattr(module, "urlopen", original)  # noqa: E731
+
+    def undo() -> None:
+        undo_opener()
+        for step in reversed(_pending.pop(name, None) or []):
+            step()
+
+    return undo
+
+
+def _flush() -> None:
+    """Route every deferred target, once proxylib can be used."""
+    if not _pending or _initializing("proxylib") or _initializing("syshttp.proxy"):
+        return
+    with _lock:
+        for name, steps in list(_pending.items()):
+            if steps is None:
+                _pending[name] = _route(name)
+
+
+class _URLErrorMap:
+    """urllib sends through one proxy: the first reachable one (``Probed``),
+    with none reachable reported as urllib reports a connection failure."""
+
+    def __init__(self, inner: Any) -> None:
+        import proxylib
+
+        self.inner = proxylib.Probed(inner)
+
+    def __getitem__(self, url: str) -> Any:
+        from urllib.error import URLError
+
+        import proxylib
+
+        try:
+            return proxylib.lookup(self.inner, url)
+        except proxylib.NoReachableProxyError as exc:
+            raise URLError(exc) from exc
+
+
+# ---- TLS: requests -----------------------------------------------------------
+
+
+def _tls_requests(requests: ModuleType) -> Callable[[], None]:
+    """Every ``HTTPAdapter`` -- proxylib's included -- verifies an https
+    origin with the system's context when ``verify`` is ``True`` and the
+    adapter's pool manager was given no context of its own."""
+    adapter_class = requests.adapters.HTTPAdapter
+    original_key = getattr(adapter_class, "build_connection_pool_key_attributes", None)
+    if original_key is None:
+        _log.debug("requests %s has no pool-key hook: its own verification stays", requests.__version__)
+        return lambda: None
+    original_verify = adapter_class.cert_verify
+
+    def ours(adapter: Any, url: str, verify: Any) -> bool:
+        pool = getattr(adapter, "poolmanager", None)
+        own = getattr(pool, "connection_pool_kw", {}).get("ssl_context")
+        return verify is True and own is None and url.lower().startswith("https")
+
+    @functools.wraps(original_key)
+    def build_connection_pool_key_attributes(self: Any, request: Any, verify: Any, cert: Any = None) -> Any:
+        host_params, pool_kwargs = original_key(self, request, verify, cert)
+        # requests 2.32.3 puts its own default context (certifi) into the pool
+        # for verify=True; later versions leave the key out.
+        requests_default = getattr(requests.adapters, "_preloaded_ssl_context", None)
+        given = pool_kwargs.get("ssl_context")
+        if ours(self, request.url or "", verify) and (given is None or given is requests_default):
+            pool_kwargs["ssl_context"] = _context()
+        return host_params, pool_kwargs
+
+    @functools.wraps(original_verify)
+    def cert_verify(self: Any, conn: Any, url: str, verify: Any, cert: Any) -> None:
+        if ours(self, url, verify):
+            # No CA file: urllib3 would load it into the shared system context.
+            original_verify(self, conn, url, False, cert)
+            conn.cert_reqs = "CERT_REQUIRED"
+            return
+        original_verify(self, conn, url, verify, cert)
+
+    setattr(adapter_class, "build_connection_pool_key_attributes", build_connection_pool_key_attributes)
+    setattr(adapter_class, "cert_verify", cert_verify)
+
+    def undo() -> None:
+        setattr(adapter_class, "build_connection_pool_key_attributes", original_key)
+        setattr(adapter_class, "cert_verify", original_verify)
+
+    return undo
+
+
+# ---- TLS: urllib -------------------------------------------------------------
+
+# One context for http.client's default: it sets ALPN on the context it is
+# handed, so it does not share the requests session's.
+_https_context: "Optional[ssl.SSLContext]" = None
+_https_context_at = 0.0
+
+
+def _default_https_context() -> ssl.SSLContext:
+    global _https_context, _https_context_at
+    from . import tls as _tls
+
+    with _lock:
+        now = time.monotonic()
+        if _https_context is None or now - _https_context_at > _tls.OS_CONTEXT_MAX_AGE:
+            _https_context = _tls.new_os_context()
+            _https_context_at = now
+        return _https_context
+
+
+def _tls_urllib(request: ModuleType) -> Callable[[], None]:
+    """``http.client``'s default HTTPS context (``ssl._create_default_https_context``)
+    becomes the system's, for urllib and every other caller that gives none."""
+    original = ssl._create_default_https_context  # type: ignore[attr-defined]
+    setattr(ssl, "_create_default_https_context", _default_https_context)
+    return lambda: setattr(ssl, "_create_default_https_context", original)
+
+
+# ---- TLS: httpx --------------------------------------------------------------
+
+
+def _tls_httpx(httpx: ModuleType) -> Callable[[], None]:
+    """``httpx.Client`` / ``AsyncClient`` built with ``verify`` left at
+    ``True`` verify with the system's context (a fresh one offering no store
+    certificate when ``cert=`` is given)."""
+
+    def wrap(client_class: Any) -> Callable[[], None]:
+        original = client_class.__init__
+
+        @functools.wraps(original)
+        def __init__(self: Any, *args: Any, **kwargs: Any) -> None:
+            if kwargs.get("verify", True) is True:
+                kwargs["verify"] = _context(certificate=kwargs.get("cert") is not None)
+            original(self, *args, **kwargs)
+
+        client_class.__init__ = __init__
+        return lambda: setattr(client_class, "__init__", original)
+
+    steps = [wrap(httpx.Client), wrap(httpx.AsyncClient)]
+
+    def undo() -> None:
+        for step in reversed(steps):
+            step()
+
+    return undo
+
+
+# ---- TLS: aiohttp ------------------------------------------------------------
+
+
+def _tls_aiohttp(aiohttp: ModuleType) -> Callable[[], None]:
+    """``ClientSession`` requests with ``ssl`` left at ``True`` and no
+    ``ssl_context`` / ``verify_ssl`` / ``fingerprint`` verify with the
+    system's context."""
+    session_class = aiohttp.ClientSession
+    original = session_class._request
+
+    @functools.wraps(original)
+    async def _request(self: Any, method: str, str_or_url: Any, **kwargs: Any) -> Any:
+        if kwargs.get("ssl", True) is True and all(
+            kwargs.get(k) is None for k in ("ssl_context", "verify_ssl", "fingerprint")
+        ):
+            kwargs["ssl"] = _context()
+        return await original(self, method, str_or_url, **kwargs)
+
+    session_class._request = _request
+    return lambda: setattr(session_class, "_request", original)
+
+
+_TLS: Dict[str, Optional[Callable[[ModuleType], Callable[[], None]]]] = {
+    "requests": _tls_requests,
+    "urllib": _tls_urllib,
+    "httpx": _tls_httpx,
+    "aiohttp": _tls_aiohttp,
+    # pip verifies with its own truststore context, which is kept.
+    "pip": None,
+}

syshttp/client/__init__.py

@@ -0,0 +1,70 @@
+"""An HTTP client on the standard library, with a replaceable transport.
+
+`HTTPClient` sends a request through a transport (an object with `send()` and
+`close()`: the default `HTTPTransport` on `http.client`, or one written
+against the `Transport` protocol) and follows redirects under one rule for
+credentials: `Authorization`, `Cookie` and `Proxy-Authorization` go to the
+origin they were given for and to no other scheme, host or port. A reply comes
+back as an `HTTPResponse`.
+
+Importing this package loads `http.client` and `ssl`. Nothing here needs a
+third-party library; `requests` in particular is not used.
+"""
+
+from __future__ import annotations
+
+import sys as _sys
+import types as _types
+
+from . import _settings
+from ._client import DEFAULT_TIMEOUT, HTTPClient
+from ._context import CA_BUNDLE
+from ._default import DEFAULT_TRANSPORT
+from ._fetch import fetch
+from ._follow import is_redirect, split_userinfo
+from ._headers import Headers, peer_text
+from ._prepare import PreparedRequest, prepare_request
+from ._response import HTTPResponse
+from ._state import COOKIE_HOSTS
+from ._status import HTTPStatusError, drop_body, json_body, raise_for_status, read_limited
+from ._transport import HTTPTransport, Transport
+
+__all__ = [
+    "CA_BUNDLE",
+    "COOKIE_HOSTS",
+    "DEFAULT_TIMEOUT",
+    "DEFAULT_TRANSPORT",
+    "HTTPClient",
+    "HTTPResponse",
+    "HTTPStatusError",
+    "HTTPTransport",
+    "Headers",
+    "MAX_REPLY_BYTES",
+    "PreparedRequest",
+    "Transport",
+    "drop_body",
+    "fetch",
+    "is_redirect",
+    "json_body",
+    "peer_text",
+    "prepare_request",
+    "raise_for_status",
+    "read_limited",
+    "split_userinfo",
+]
+
+
+class _Module(_types.ModuleType):
+    """The module, with `MAX_REPLY_BYTES` read from and written to where it
+    takes effect."""
+
+    @property
+    def MAX_REPLY_BYTES(self) -> int:
+        return _settings.MAX_REPLY_BYTES
+
+    @MAX_REPLY_BYTES.setter
+    def MAX_REPLY_BYTES(self, value: int) -> None:
+        _settings.MAX_REPLY_BYTES = value
+
+
+_sys.modules[__name__].__class__ = _Module

syshttp/client/_body.py

@@ -0,0 +1,156 @@
+"""How a request body is framed: the length it is sent under, and a body that
+sends exactly that many bytes."""
+
+from __future__ import annotations
+
+import errno as _errno
+import os as _os
+import stat as _stat
+import typing as _ty
+
+_STEP = 65536
+
+
+def body_length(body) -> "int | None":
+    """The bytes `body` will send from now on, when that is known without
+    reading it, else `None`: a bytes-like object, a list or tuple of bytes, a
+    file that is seekable (from `tell()` to its end, the position restored) or
+    is a regular file (its size less `tell()`). An iterator, a generator and a
+    stream that cannot report its size have no length."""
+    if body is None:
+        return 0
+    if isinstance(body, (bytes, bytearray, memoryview)):
+        return memoryview(body).nbytes
+    if isinstance(body, (list, tuple)):
+        if all(isinstance(piece, (bytes, bytearray)) for piece in body):
+            return sum(len(piece) for piece in body)
+        return None
+    if hasattr(body, "read"):
+        return _file_length(body)
+    return None
+
+
+def _file_length(file) -> "int | None":
+    try:
+        seekable = file.seekable()
+    except (AttributeError, ValueError, OSError):
+        seekable = False
+    if seekable:
+        here = file.tell()
+        try:
+            end = file.seek(0, 2)
+            if end is None:
+                end = file.tell()
+        finally:
+            file.seek(here)
+        return max(0, end - here)
+    try:
+        info = _os.fstat(file.fileno())
+        if _stat.S_ISREG(info.st_mode):
+            return max(0, info.st_size - file.tell())
+    except (AttributeError, ValueError, OSError):
+        pass
+    return None
+
+
+def declared_length(pairs: "_ty.Iterable[tuple[str, str]]") -> "int | None":
+    """The `Content-Length` the caller set, or `None`; `ValueError` for one
+    that is repeated or is not a count of bytes."""
+    found = [value for name, value in pairs if name.lower() == "content-length"]
+    if not found:
+        return None
+    value = found[0].strip()
+    if len(found) > 1 or not (value.isascii() and value.isdigit()):
+        raise ValueError("the Content-Length header is not one count of bytes")
+    return int(value)
+
+
+class Sized:
+    """A body sent under a `Content-Length`: exactly `length` bytes of `source`
+    (a binary file or an iterable of bytes). It never reads or sends more, so a
+    file that grew since it was measured cannot overrun the length, and a
+    source that ends first is `OSError(EIO)`, not a shorter upload."""
+
+    def __init__(self, source, length: int):
+        self._length = length
+        self._left = length
+        if hasattr(source, "read"):
+            self._read = source.read
+        else:
+            self._pieces = iter(source)
+            self._pending = b""
+            self._read = self._read_pieces
+
+    def __len__(self) -> int:
+        return self._length
+
+    def __iter__(self) -> "_ty.Iterator[bytes]":
+        while True:
+            chunk = self.read(_STEP)
+            if not chunk:
+                return
+            yield chunk
+
+    def read(self, amount: "int | None" = -1) -> bytes:
+        if self._left <= 0:
+            return b""
+        want = self._left if amount is None or amount < 0 else min(amount, self._left)
+        chunk = self._read(want)
+        if isinstance(chunk, str):
+            raise TypeError("the request body is bytes, not text")
+        if not chunk:
+            raise OSError(_errno.EIO, "the request body ended before its Content-Length")
+        chunk = bytes(chunk[:want])
+        self._left -= len(chunk)
+        return chunk
+
+    def _read_pieces(self, want: int) -> bytes:
+        while not self._pending:
+            try:
+                self._pending = bytes(next(self._pieces))
+            except StopIteration:
+                return b""
+        chunk, self._pending = self._pending[:want], self._pending[want:]
+        return chunk
+
+
+def _chunks(file) -> "_ty.Iterator[bytes]":
+    while True:
+        chunk = file.read(_STEP)
+        if not chunk:
+            return
+        yield chunk
+
+
+def prepare_body(body, pairs: "list[tuple[str, str]]"):
+    """`(value, length)`: what to hand the HTTP layer as the body, and the
+    `Content-Length` to add (`None` when the request already names one, the
+    layer adds its own, or none is needed). A length the caller set is
+    trusted, so nothing is chunked and exactly that many bytes are sent;
+    otherwise a body whose length is known is sent under it and any other is
+    left for the layer to send chunked."""
+    if body is None:
+        return None, None
+    declared = declared_length(pairs)
+    known = body_length(body)
+    if isinstance(body, (bytes, bytearray, memoryview)):
+        if declared is None or declared == known:
+            return body, None
+        if known is not None and declared > known:
+            raise ValueError("the Content-Length header is larger than the body")
+        return bytes(memoryview(body)[:declared]), None
+    length: "int | None"
+    add: "int | None"
+    if declared is not None:
+        if known is not None and declared > known:
+            raise ValueError("the Content-Length header is larger than the body")
+        length, add = declared, None
+    elif any(name.lower() == "transfer-encoding" for name, _ in pairs):
+        return body, None  # the caller frames it
+    else:
+        length, add = known, known
+    if length is None:
+        return (_chunks(body) if hasattr(body, "read") else body), None
+    if length == 0:
+        return b"", add
+    return Sized(body, length), add