syshttp 0.1.0__py3-none-any.whl

syshttp/client/_fetch.py

@@ -0,0 +1,98 @@
+"""`fetch()`: one request, one response, through a client made for the call or
+one the caller keeps."""
+
+from __future__ import annotations
+
+import collections.abc as _abc
+import json as _json
+import typing as _ty
+import urllib.parse as _urlparse
+
+from ._client import DEFAULT_TIMEOUT, HTTPClient
+from ._headers import has_header
+from ._response import HTTPResponse
+
+REAUTH_STATUSES = (401, 403)
+"""The statuses after which `fetch(reauth=...)` runs `reauth` and sends again."""
+
+
+def _body(data: _ty.Any, json: _ty.Any, headers: "_ty.Mapping[str, str] | None") -> "tuple[_ty.Any, dict[str, str]]":
+    """`data` or `json` as the body to send, and `headers` with the
+    `Content-Type` that goes with it unless they name one."""
+    if data is not None and json is not None:
+        raise ValueError("data and json are two bodies: give one")
+    sent = dict(headers or {})
+    if json is not None:
+        body: _ty.Any = _json.dumps(json).encode("utf-8")
+        kind = "application/json"
+    elif isinstance(data, _abc.Mapping):
+        body = _urlparse.urlencode(data, doseq=True).encode("ascii")
+        kind = "application/x-www-form-urlencoded"
+    else:
+        return data, sent
+    if not has_header(sent, "Content-Type"):
+        sent["Content-Type"] = kind
+    return body, sent
+
+
+def _reauthenticate(reauth: _ty.Any, client: HTTPClient, url: str) -> None:
+    ensure = getattr(reauth, "ensure", None)
+    if callable(ensure):
+        ensure(client, url)
+    else:
+        reauth(client, url)
+
+
+def fetch(
+    url: str,
+    method: str = "GET",
+    *,
+    headers: "_ty.Mapping[str, str] | None" = None,
+    data: _ty.Any = None,
+    json: _ty.Any = None,
+    timeout: "float | tuple | None" = DEFAULT_TIMEOUT,
+    client: "HTTPClient | None" = None,
+    transport: _ty.Any = None,
+    reauth: _ty.Any = None,
+    **client_options: _ty.Any,
+) -> HTTPResponse:
+    """Send `method` to `url` and return the response with its body read whole
+    (`MAX_REPLY_BYTES` bounds it), so the caller takes what it needs from it:
+    `.status`, `.headers`, `.content`, `.text()` or `.json()`. A status is never
+    an exception: `raise_for_status()` is the caller's choice.
+
+    `data` is bytes, text, a mapping (form-encoded) or a binary file; `json` a
+    value sent as JSON (`None` sends none: use `data=b"null"` for that);
+    giving both is `ValueError`. `timeout` is seconds or a `(connect, read)`
+    pair. The request goes through `client` when given (left open), else
+    through a new `HTTPClient(transport, **client_options)` closed after the
+    call: `transport` is any object implementing the transport protocol (`None`:
+    the `SYSHTTP_DEFAULT_TRANSPORT` setting's, else syshttp's own), and
+    `client_options` are the other keywords of `HTTPClient` (`env`, `proxymap`,
+    `ssl_context`, `retries`, ...). A `client` beside `transport` or
+    `client_options` is `ValueError`: a given client already has them.
+
+    `reauth` is called as `reauth(client, url)` (an object with
+    `ensure(client, url)`, an `AuthProvider`, is called through that) once after
+    a 401 or 403, and the request is sent one more time; its errors are the
+    caller's. The body must then be one that can be sent twice (bytes, text, a
+    mapping or `json`)."""
+    if client is not None and (transport is not None or client_options):
+        raise ValueError(
+            "a client was given, and it has its transport and settings already: "
+            "pass transport and client options to fetch() alone, or set them on the client"
+        )
+    body, sent = _body(data, json, headers)
+    if reauth is not None and not (body is None or isinstance(body, (bytes, bytearray, memoryview, str))):
+        raise ValueError("reauth sends the request twice: the body must be bytes, text, a mapping or json")
+    own = client is None
+    used = HTTPClient(transport, **client_options) if client is None else client
+    try:
+        response = used.request(method, url, data=body, headers=sent, timeout=timeout)
+        if reauth is not None and response.status in REAUTH_STATUSES:
+            _reauthenticate(reauth, used, url)
+            response = used.request(method, url, data=body, headers=sent, timeout=timeout)
+        return response
+    finally:
+        if own:
+            used.close()

syshttp/client/_follow.py

@@ -0,0 +1,243 @@
+"""What a backend does above a transport: credentials, redirects and the
+errors it raises for a redirect it will not follow."""
+
+from __future__ import annotations
+
+import base64 as _base64
+import errno as _errno
+import re as _re
+import urllib.parse as _urlparse
+
+from ._headers import check_url, drop_header, peer_text, shown
+from ._response import HTTPResponse
+
+MAX_REDIRECTS = 30
+"""Most redirects a `GET`, `HEAD` or `OPTIONS` follows."""
+
+CREDENTIAL_HEADERS = ("Authorization", "Cookie", "Proxy-Authorization")
+"""The headers a caller keeps a secret in, which a redirect to another scheme,
+host or port leaves off, for that hop and every later one. The cookies of the
+client's jar are chosen again for each hop."""
+
+FOLLOWED_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
+"""Methods whose redirects are followed. Every other method goes through
+`send_once()`: following would turn a state-changing one into a different
+request, and a PROPFIND into a GET or one without its body."""
+
+RESENT_STATUSES = {"PROPFIND": (301, 302, 303, 307, 308)}
+"""The redirect statuses after which a method is sent again, same method and
+body, to a Location on the same scheme, host and port. PROPFIND only reads,
+so it is repeated for every redirect; any other method only for 307 and 308,
+which are defined to keep the method and the body."""
+
+_REDIRECTS = (301, 302, 303, 307, 308)
+_DEFAULT_PORTS = {"http": 80, "https": 443}
+_RE_URL_SCHEME = _re.compile(r"[A-Za-z][A-Za-z0-9+.-]*://")
+_DRAIN = 64 * 1024
+
+
+def credential(text: str) -> "str | bytes":
+    """The octets a percent-encoded userinfo part stands for (RFC 3986 2.1),
+    as `str` when they are ASCII and as `bytes` otherwise."""
+    octets = _urlparse.unquote_to_bytes(text)
+    return octets.decode("ascii") if octets.isascii() else octets
+
+
+def split_userinfo(url: str) -> "tuple[str, tuple[str|bytes, str|bytes] | None]":
+    """Split the userinfo out of an absolute URL: `(url_without_userinfo,
+    (user, password) | None)`, both parts percent-decoded to the octets they
+    stand for. Credentials inside a request URL would end up in
+    `HTTPResponse.url`, in error text and in redirect handling; they are sent
+    in an `Authorization` header instead."""
+    match = _RE_URL_SCHEME.match(url)
+    if not match:
+        return url, None
+    start = match.end()
+    end = len(url)
+    for sep in "/?#":
+        index = url.find(sep, start)
+        if index != -1 and index < end:
+            end = index
+    userinfo, at, hostport = url[start:end].rpartition("@")
+    if not at:
+        return url, None
+    user, _, password = userinfo.partition(":")
+    auth = (credential(user), credential(password))
+    return url[:start] + hostport + url[end:], (auth if any(auth) else None)
+
+
+def basic_authorization(auth: "tuple[str | bytes, str | bytes]") -> str:
+    """The `Authorization` value for Basic credentials (RFC 7617)."""
+    user, password = (p.encode("utf-8") if isinstance(p, str) else p for p in auth)
+    return "Basic " + _base64.b64encode(user + b":" + password).decode("ascii")
+
+
+def origin(url: str) -> "tuple[str, str, int | None]":
+    """The scheme, host and port a URL names."""
+    split = _urlparse.urlsplit(url)
+    scheme = split.scheme.lower()
+    try:
+        port = split.port
+    except ValueError:
+        port = -1
+    # `hostname` is lower case already; a host name is case-insensitive.
+    return scheme, (split.hostname or "").lower(), port or _DEFAULT_PORTS.get(scheme)
+
+
+def is_redirect(resp: HTTPResponse) -> bool:
+    return resp.status in _REDIRECTS and "Location" in resp.headers
+
+
+def redirect_location(url: str, resp: HTTPResponse) -> "str | None":
+    """`resp`'s `Location` as an absolute `http:`/`https:` URL without
+    userinfo, or `None` when there is none or it cannot be used."""
+    location = resp.headers.get("Location")
+    if not location:
+        return None
+    try:
+        # http.client decodes header bytes as Latin-1; servers send UTF-8.
+        location = location.encode("latin1").decode("utf-8")
+    except (UnicodeEncodeError, UnicodeDecodeError):
+        pass
+    if any(ord(c) < 0x20 or ord(c) == 0x7F for c in location):
+        return None
+    try:
+        absolute = split_userinfo(_urlparse.urljoin(url, location))[0]
+        # What a transport would refuse is not a place to go.
+        check_url(absolute)
+        return absolute
+    except ValueError:
+        return None
+
+
+def refused_redirect(method: str, url: str, resp: HTTPResponse, location, reason):
+    status = f"HTTP {resp.status} {peer_text(resp.reason or '')}".rstrip()
+    where = peer_text(shown(location), 200) if location else "(none)"
+    return OSError(
+        _errno.EIO,
+        f"{status} for {method} {shown(url)}: redirect to {where} " f"not followed ({reason})",
+    )
+
+
+def discard(resp: HTTPResponse) -> None:
+    """Gives up the rest of a body nobody wants, reading a small one so the
+    connection can serve the next request."""
+    try:
+        resp.read(_DRAIN)
+    except OSError:
+        pass
+    finally:
+        resp.close()
+
+
+def _capped(timeout, deadline):
+    """`timeout` held to what remains of `deadline`; as it is without one."""
+    return timeout if deadline is None else deadline.cap(timeout)
+
+
+def can_resend_body(body) -> bool:
+    return body is None or isinstance(body, (bytes, bytearray, memoryview))
+
+
+def send_following(
+    transport,
+    method: str,
+    url: str,
+    *,
+    headers: "dict[str, str]",
+    timeout,
+    stream: bool,
+    first: "str | None" = None,
+    deadline=None,
+) -> HTTPResponse:
+    """Send a request that carries no body and follow its redirects, up to
+    `MAX_REDIRECTS`. `Authorization`, `Cookie` and `Proxy-Authorization` go
+    to the origin of `first` (the URL the caller asked for; `url` when it is
+    not given) and nowhere else: a request to another scheme, host or port
+    drops them, and for every later hop. A `303` becomes a `GET` (unless the
+    request was a `HEAD`)."""
+    home = origin(first or url)
+    current = url
+    hops = 0
+    while True:
+        if origin(current) != home:
+            for name in CREDENTIAL_HEADERS:
+                headers = drop_header(headers, name)
+        resp = transport.send(
+            method,
+            current,
+            headers=headers,
+            body=None,
+            timeout=_capped(timeout, deadline),
+            stream=stream,
+        )
+        if not is_redirect(resp):
+            return resp
+        location = redirect_location(current, resp)
+        if location is None:
+            discard(resp)
+            raise refused_redirect(method, current, resp, None, "no usable Location")
+        if hops >= MAX_REDIRECTS:
+            discard(resp)
+            raise OSError(
+                _errno.EIO,
+                f"more than {MAX_REDIRECTS} redirects from {shown(url)}",
+            )
+        discard(resp)
+        if resp.status == 303 and method != "HEAD":
+            method = "GET"
+        current = location
+        hops += 1
+
+
+def send_once(
+    transport,
+    method: str,
+    url: str,
+    *,
+    headers: "dict[str, str]",
+    body,
+    timeout,
+    stream: bool,
+    deadline=None,
+) -> HTTPResponse:
+    """Send a request that must not be redirected the way a browser does: a
+    redirect with a status in `RESENT_STATUSES` to the same scheme, host and
+    port is sent once more with the same method and body, and any other 3xx
+    raises `OSError(EIO)` naming the status and the `Location`."""
+    resp = transport.send(
+        method,
+        url,
+        headers=headers,
+        body=body,
+        timeout=_capped(timeout, deadline),
+        stream=stream,
+    )
+    if not 300 <= resp.status < 400:
+        return resp
+    location = redirect_location(url, resp)
+    resent = RESENT_STATUSES.get(method.upper(), (307, 308))
+    if resp.status not in resent:
+        reason = "only " + ", ".join(map(str, resent[:-1])) + f" and {resent[-1]} send the request again"
+    elif location is None:
+        reason = "no usable Location"
+    elif origin(location) != origin(url):
+        reason = "it leaves the scheme, host and port of the request"
+    elif not can_resend_body(body):
+        reason = "the request body cannot be sent twice"
+    else:
+        discard(resp)
+        resp = transport.send(
+            method,
+            location,
+            headers=headers,
+            body=body,
+            timeout=_capped(timeout, deadline),
+            stream=stream,
+        )
+        if not 300 <= resp.status < 400:
+            return resp
+        url, location = location, redirect_location(location, resp)
+        reason = "a second redirect"
+    discard(resp)
+    raise refused_redirect(method, url, resp, location, reason)

syshttp/client/_headers.py

@@ -0,0 +1,238 @@
+"""Headers and the checks a request passes before anything is sent."""
+
+from __future__ import annotations
+
+import collections.abc as _abc
+import math as _math
+import re as _re
+import typing as _ty
+import urllib.parse as _urlparse
+
+_TOKEN = _re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
+_CONTROL = _re.compile(r"[\x00-\x08\x0a-\x1f\x7f]")
+_URL_CONTROL = _re.compile(r"[\x00-\x1f\x7f]")
+_NOT_PRINTABLE = _re.compile(r"[^\x21-\x7e]")
+
+_DEFAULT_PORTS = {"http": 80, "https": 443}
+
+
+class Headers(_abc.Mapping):
+    """The headers of a response: a read-only mapping whose names compare
+    without regard to case. A name sent more than once reads as its values
+    joined with `", "` (RFC 9110 5.3); `get_all()` returns them separately."""
+
+    __slots__ = ("_pairs", "_joined", "_spelling")
+
+    def __init__(self, pairs: "_ty.Iterable[tuple[str, str]] | _abc.Mapping" = ()):
+        if isinstance(pairs, _abc.Mapping):
+            pairs = pairs.items()
+        self._pairs = tuple((str(name), str(value)) for name, value in pairs)
+        grouped: "dict[str, list[str]]" = {}
+        self._spelling: "dict[str, str]" = {}
+        for name, value in self._pairs:
+            key = name.lower()
+            grouped.setdefault(key, []).append(value)
+            self._spelling.setdefault(key, name)
+        self._joined = {key: ", ".join(values) for key, values in grouped.items()}
+
+    def __getitem__(self, name: str) -> str:
+        return self._joined[name.lower()]
+
+    def __iter__(self) -> "_ty.Iterator[str]":
+        return iter(self._spelling.values())
+
+    def __len__(self) -> int:
+        return len(self._joined)
+
+    def __contains__(self, name: object) -> bool:
+        return isinstance(name, str) and name.lower() in self._joined
+
+    def get_all(self, name: str) -> "list[str]":
+        """Every value sent for `name`, in order."""
+        key = name.lower()
+        return [value for field, value in self._pairs if field.lower() == key]
+
+    def __repr__(self) -> str:
+        # Values are left out: a reply can carry cookies and tokens.
+        return f"Headers({list(self._spelling.values())!r})"
+
+
+def peer_text(text: object, limit: int = 120) -> str:
+    """`text` that came from the other end (a reason phrase, a `Location`, a
+    header value) as an error message may show it: characters a terminal or a
+    log would act on (controls, line and bidirectional separators) escaped, and
+    cut to `limit` characters."""
+    text = str(text)
+    cut = len(text) > limit
+    shown = "".join(
+        char if char.isprintable() else char.encode("unicode_escape").decode("ascii") for char in text[:limit]
+    )
+    return shown + "..." if cut else shown
+
+
+def shown(url: str) -> str:
+    """`url` as an error message may print it: scheme, host, port and path; no
+    userinfo, query or fragment, which can carry a credential."""
+    try:
+        split = _urlparse.urlsplit(url)
+        netloc = split.netloc.rpartition("@")[2]
+        return _urlparse.urlunsplit((split.scheme, netloc, split.path, "", ""))
+    except ValueError:
+        return "(unreadable URL)"
+
+
+def has_header(headers: "_abc.Mapping[str, str]", name: str) -> bool:
+    key = name.lower()
+    return any(field.lower() == key for field in headers)
+
+
+def wants_decoding(pairs: "_ty.Iterable[tuple[str, str]]") -> bool:
+    """Whether a response is decoded for a request with these headers: unless
+    the request names a content coding of its own in `Accept-Encoding`
+    (`identity` is not one, so a server that encodes anyway is still decoded)."""
+    for name, value in pairs:
+        if name.lower() == "accept-encoding":
+            tokens = {part.split(";")[0].strip().lower() for part in value.split(",")}
+            return tokens <= {"", "identity"}
+    return True
+
+
+def drop_header(headers: "_abc.Mapping[str, str]", name: str) -> "dict[str, str]":
+    """`headers` as a new dict without `name`, whatever its case."""
+    key = name.lower()
+    return {field: value for field, value in headers.items() if field.lower() != key}
+
+
+def merge_headers(*layers: "_abc.Mapping[str, str] | None") -> "dict[str, str]":
+    """The layers laid over one another, a later one winning on a name whatever
+    its case; a value of `None` removes the name."""
+    merged: "dict[str, tuple[str, str]]" = {}
+    for layer in layers:
+        for name, value in (layer or {}).items():
+            if value is None:
+                merged.pop(name.lower(), None)
+            else:
+                merged[name.lower()] = (name, value)
+    return dict(merged.values())
+
+
+def check_timeout(timeout) -> "tuple[float | None, float | None]":
+    """`timeout` (a number, `None`, or a `(connect, read)` pair) as that pair;
+    `ValueError` for anything else."""
+    if timeout is None:
+        return None, None
+    if isinstance(timeout, tuple):
+        if len(timeout) != 2:
+            raise ValueError("a timeout is a number, None or a (connect, read) pair")
+        connect, read = timeout
+    else:
+        connect = read = timeout
+    for value in (connect, read):
+        if value is not None and (
+            isinstance(value, bool) or not isinstance(value, (int, float)) or not 0 < value < _math.inf
+        ):
+            raise ValueError("a timeout is a positive finite number of seconds, or None")
+    return connect, read
+
+
+def check_method(method: str) -> str:
+    if not isinstance(method, str) or not _TOKEN.fullmatch(method):
+        raise ValueError("the request method is not an HTTP token")
+    return method.upper()
+
+
+def check_body(body) -> bool:
+    """Whether `body` (`None`, bytes, a binary file or an iterable of bytes)
+    can be sent twice; `TypeError` for text or anything else."""
+    if body is None or isinstance(body, (bytes, bytearray, memoryview)):
+        return True
+    if isinstance(body, str):
+        raise TypeError("the request body is bytes, not text")
+    if hasattr(body, "read"):
+        return False
+    if isinstance(body, (list, tuple)):
+        return all(isinstance(piece, (bytes, bytearray)) for piece in body)
+    if hasattr(body, "__iter__"):
+        return False
+    raise TypeError("the request body is bytes, a file or an iterable of bytes")
+
+
+def check_headers(headers: "_abc.Mapping[str, str] | None") -> "list[tuple[str, str]]":
+    """`headers` as `(name, value)` text pairs, or `ValueError` for a name that
+    is not an HTTP token or a value with a control character, a line break
+    included. The refused text is not quoted: it may be a credential."""
+    pairs = []
+    for name, value in (headers or {}).items():
+        if not isinstance(name, str) or not _TOKEN.fullmatch(name):
+            raise ValueError("a request header name is not an HTTP token")
+        if isinstance(value, bytes):
+            value = value.decode("latin-1")
+        elif isinstance(value, int) and not isinstance(value, bool):
+            value = str(value)
+        if not isinstance(value, str):
+            raise ValueError(f"the value of the {name} header is not text")
+        if _CONTROL.search(value):
+            raise ValueError(f"the value of the {name} header has a control character")
+        try:
+            value.encode("latin-1")
+        except UnicodeEncodeError:
+            raise ValueError(f"the value of the {name} header is not Latin-1") from None
+        pairs.append((name, value))
+    return pairs
+
+
+class Target(_ty.NamedTuple):
+    """Where a request goes and how it names itself on the wire."""
+
+    scheme: str
+    host: str  # IDNA, without brackets
+    port: int
+    path: str  # the origin-form request target, percent-encoded
+    authority: str  # the Host header value
+
+
+def _percent(match: "_re.Match") -> str:
+    return "".join(f"%{byte:02X}" for byte in match.group().encode("utf-8", "surrogateescape"))
+
+
+def bracket(host: str) -> str:
+    return f"[{host}]" if ":" in host else host
+
+
+def check_url(url: str) -> Target:
+    """The parts of an `http:`/`https:` URL a request is made of. A control
+    character (CR and LF included), credentials in the URL, another scheme or
+    a missing host are `ValueError`; the URL is not quoted in the message."""
+    if not isinstance(url, str):
+        raise ValueError("the request URL is not text")
+    if _URL_CONTROL.search(url):
+        raise ValueError("the request URL has a control character")
+    try:
+        parts = _urlparse.urlsplit(url)
+        port = parts.port
+    except ValueError:
+        raise ValueError("the request URL cannot be parsed") from None
+    scheme = parts.scheme.lower()
+    if scheme not in _DEFAULT_PORTS:
+        raise ValueError(f"the request URL has scheme {scheme!r}, not http or https")
+    if "@" in parts.netloc:
+        raise ValueError("the request URL carries credentials; send them in an Authorization header")
+    # The connection pool, the Host header, SNI and the proxy rules key on this
+    # name, which is case-insensitive: `hostname` is lower case already.
+    host = (parts.hostname or "").lower()
+    if not host:
+        raise ValueError("the request URL has no host")
+    if not host.isascii():
+        try:
+            host = host.encode("idna").decode("ascii")
+        except UnicodeError:
+            raise ValueError("the host of the request URL is not a valid name") from None
+    port = port or _DEFAULT_PORTS[scheme]
+    path = parts.path or "/"
+    if parts.query:
+        path += "?" + parts.query
+    path = _NOT_PRINTABLE.sub(_percent, path)
+    authority = bracket(host)
+    if port != _DEFAULT_PORTS[scheme]:
+        authority += f":{port}"
+    return Target(scheme, host, port, path, authority)