pathapi 0.1.0__py3-none-any.whl

pathapi/http/_client.py

@@ -0,0 +1,211 @@
+"""The client: a transport, the headers and timeout every request is sent with,
+and the redirect rules above the transport."""
+
+from __future__ import annotations
+
+import threading as _threading
+import typing as _ty
+
+from .. import __version__
+from . import _follow, _settings
+from ._headers import check_headers, check_url, has_header, merge_headers
+from ._response import HTTPResponse
+from ._status import too_large
+from ._transports import default_transport
+
+DEFAULT_TIMEOUT = (10, 60)
+"""`(connect, read)` timeout, in seconds, a client sends with every request
+unless it is given `timeout` (`HTTPClient(timeout=...)` or per request);
+`timeout=None` waits without a limit."""
+
+_UNSET = object()
+"""The `timeout` of a call that did not give one: the client's own."""
+
+
+class HTTPClient:
+    """A transport with the settings every request is sent with, and the rules
+    for what happens after a redirect.
+
+    `transport` is any object with a `send()` method as `HTTPTransport.send()`;
+    without one the client builds the transport that `PATHAPI_HTTP_TRANSPORT`
+    names (the standard-library `HTTPTransport` by default) on first use and
+    closes it with `close()` or at the end of a `with` block. A transport it
+    was given stays the caller's. `headers` are sent with every request,
+    request-specific headers merged over them by name, over a `User-Agent` of
+    `user_agent` (`pathapi/<version>` by default) and `Accept: */*`.
+    Every request gets `timeout=DEFAULT_TIMEOUT` (`(10, 60)` seconds) unless
+    `timeout` here or in the call says otherwise (`None` waits without limit).
+
+    The client, not the transport, follows redirects. `GET`, `HEAD` and
+    `OPTIONS` follow up to 30; a redirect that changes the scheme, host or
+    port drops `Authorization` (and nothing else). Any other method is sent
+    without following: a 307 or 308 (for `PROPFIND` also a 301, 302 or 303) to
+    the same scheme, host and port is sent once more with the same method and
+    body, and any other 3xx, a redirect to another origin or a second redirect
+    raises `OSError(EIO)`."""
+
+    __slots__ = (
+        "headers",
+        "timeout",
+        "user_agent",
+        "_transport",
+        "_built",
+        "_transport_lock",
+        "__weakref__",
+    )
+
+    def __init__(
+        self,
+        transport: "_ty.Any" = None,
+        *,
+        headers: "_ty.Mapping[str, str] | None" = None,
+        timeout: "float | tuple | None" = DEFAULT_TIMEOUT,
+        user_agent: "str | None" = None,
+    ):
+        if transport is not None and not callable(getattr(transport, "send", None)):
+            raise TypeError("a transport is an object with a send() method")
+        self._transport = transport
+        self._built = None
+        self._transport_lock = _threading.Lock()
+        self.headers = dict(headers or {})
+        self.timeout = timeout
+        self.user_agent = user_agent or f"pathapi/{__version__}"
+
+    def __repr__(self) -> str:
+        # Headers are left out: they carry credentials.
+        return f"{type(self).__name__}()"
+
+    @property
+    def transport(self):
+        """The transport: the one the client was given, or one built on first
+        use. Read once per request."""
+        transport = self._transport
+        if transport is None:
+            with self._transport_lock:
+                if self._transport is None:
+                    self._built = self._transport = default_transport()
+                transport = self._transport
+        return transport
+
+    def close(self) -> None:
+        """Closes the transport this client built for itself; one it was given
+        stays the caller's. Harmless when called again."""
+        with self._transport_lock:
+            built, self._built = self._built, None
+            if built is not None:
+                self._transport = None
+        if built is not None:
+            built.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_info) -> None:
+        self.close()
+
+    def _sent_headers(self, headers, auth) -> "dict[str, str]":
+        """The headers every request starts from: the base ones, the client's,
+        the call's, and the `Authorization` that `auth` stands for."""
+        base = {"User-Agent": self.user_agent, "Accept": "*/*"}
+        sent = merge_headers(base, self.headers, headers)
+        if auth is not None and not has_header(sent, "Authorization"):
+            sent["Authorization"] = (
+                auth if isinstance(auth, str) else _follow.basic_authorization(auth)
+            )
+        return sent
+
+    def _dispatch(
+        self, method, url, sent, data, allow_redirects, stream, timeout, first=None
+    ) -> HTTPResponse:
+        check_url(url)
+        check_headers(sent)
+        if timeout is _UNSET:
+            timeout = self.timeout
+        if isinstance(data, str):
+            data = data.encode("utf-8")
+        transport = self.transport
+        # Always streamed from the transport, so that no reply is read whole
+        # before it is known to be small enough (see `stream` below).
+        if method.upper() in _follow.FOLLOWED_METHODS and data is None:
+            if allow_redirects is False and first is None:
+                resp = transport.send(
+                    method, url, headers=sent, timeout=timeout, stream=True
+                )
+            else:
+                resp = _follow.send_following(
+                    transport,
+                    method,
+                    url,
+                    headers=sent,
+                    timeout=timeout,
+                    stream=True,
+                    first=first,
+                )
+        else:
+            resp = _follow.send_once(
+                transport,
+                method,
+                url,
+                headers=sent,
+                body=data,
+                timeout=timeout,
+                stream=True,
+            )
+        if not stream:
+            limit = _settings.MAX_REPLY_BYTES
+            resp._buffered(limit, too_large(_follow.shown(url), limit))
+        return resp
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        data=None,
+        headers: "_ty.Mapping[str, str] | None" = None,
+        stream: bool = False,
+        allow_redirects: "bool | None" = None,
+        timeout=_UNSET,
+        auth: "tuple[str | bytes, str | bytes] | str | None" = None,
+    ) -> HTTPResponse:
+        """Send `method` to `url` and return the final response. A `data` that
+        is not `bytes` (a file, an iterable) cannot be sent twice, so it is not
+        re-sent after a redirect or a dropped connection.
+        `allow_redirects=False` returns the first answer of a `GET`, `HEAD` or
+        `OPTIONS` as it is. `auth` is a `(user, password)` pair sent as
+        `Authorization: Basic`, or a `str` sent as the `Authorization` value
+        (`"Bearer <token>"`), to the origin of `url` unless a header names an
+        `Authorization`. `ValueError` for a URL (credentials in it included)
+        or a header with a control character, before anything is sent.
+
+        With `stream=False` the body is read before this returns, and a decoded
+        body over `MAX_REPLY_BYTES` is `OSError(EFBIG)`; with `stream=True` the
+        caller reads it, and closes the response."""
+        sent = self._sent_headers(headers, auth)
+        return self._dispatch(method, url, sent, data, allow_redirects, stream, timeout)
+
+    def redirect(
+        self,
+        method: str,
+        url: str,
+        resp: HTTPResponse,
+        *,
+        headers: "_ty.Mapping[str, str] | None" = None,
+        stream: bool = False,
+        timeout=_UNSET,
+        auth: "tuple[str | bytes, str | bytes] | str | None" = None,
+    ) -> HTTPResponse:
+        """Send `method` to where `resp`, the answer to a request for `url`,
+        redirects to, and follow what it redirects to in turn. The request
+        carries what `url` is sent with, except that `Authorization` is left
+        off whenever it goes to another scheme, host or port than `url`'s. A
+        redirect with no usable `Location` is `OSError(EIO)`."""
+        sent = self._sent_headers(headers, auth)
+        location = _follow.redirect_location(url, resp)
+        if location is None:
+            raise _follow.refused_redirect(
+                method, url, resp, None, "no usable Location"
+            )
+        return self._dispatch(
+            method, location, sent, None, None, stream, timeout, first=url
+        )

pathapi/http/_errors.py

@@ -0,0 +1,79 @@
+"""How a failed request becomes the `OSError` a path raises."""
+
+from __future__ import annotations
+
+import contextlib as _contextlib
+import errno as _errno
+import os as _os
+
+from ._headers import peer_text
+from ._status import HTTPStatusError as _HTTPStatusError
+
+_ERRNOS = {
+    FileNotFoundError: _errno.ENOENT,
+    PermissionError: _errno.EACCES,
+    FileExistsError: _errno.EEXIST,
+    IsADirectoryError: _errno.EISDIR,
+    NotADirectoryError: _errno.ENOTDIR,
+}
+
+
+def os_error(error_class, subject) -> OSError:
+    """`error_class` built as pathlib builds it -- `(errno, strerror,
+    filename)` -- for `subject` (a path or a URL, shown through `str()`), so
+    `e.errno` and `e.filename` are set."""
+    code = _ERRNOS.get(error_class, _errno.EIO)
+    return error_class(code, _os.strerror(code), str(subject))
+
+
+def server_size(text: "str | None") -> "int | None":
+    """A byte count a server stated (`Content-Length`, `getcontentlength`) as
+    an int, or `None` when it is not a plain non-negative decimal number."""
+    text = (text or "").strip()
+    if text.isascii() and text.isdigit() and len(text) <= 19:
+        return int(text)
+    return None
+
+
+@_contextlib.contextmanager
+def translate_errors(subject, *, conflict=FileExistsError):
+    """Map a failed request to the exception `pathlib` raises for `subject`.
+
+    An `HTTPStatusError` becomes `FileNotFoundError` (404, 410),
+    `PermissionError` (401, 403; 405 and 501 too, naming the status),
+    `conflict` (409) or `OSError(EIO)` naming the status; a `TimeoutError` or
+    `ConnectionError` from the transport is raised again with a message that
+    holds `subject` and the exception's type name. `conflict` is what a 409
+    means: for a write (PUT, MKCOL, MOVE) RFC 4918 uses 409 for a missing
+    parent collection, so writers pass `FileNotFoundError`. The cause is not
+    chained: a traceback would otherwise print a URL, a header or a
+    credential that the transport's own error carries."""
+    try:
+        yield
+    except _HTTPStatusError as e:
+        response = e.response
+        status = response.status
+        reason = peer_text(response.reason or "")
+        if status in (404, 410):
+            raise os_error(FileNotFoundError, subject) from None
+        elif status in (401, 403):
+            raise os_error(PermissionError, subject) from None
+        elif status == 409:
+            raise os_error(conflict, subject) from None
+        elif status in (405, 501):
+            raise PermissionError(
+                _errno.EACCES,
+                f"Method not allowed (HTTP {status})",
+                str(subject),
+            ) from None
+        else:
+            raise OSError(
+                _errno.EIO,
+                f"HTTP Error {status} {reason}".rstrip() + f" for {subject}",
+            ) from None
+    except TimeoutError as e:
+        raise TimeoutError(f"Timeout for {subject} ({type(e).__name__})") from None
+    except ConnectionError as e:
+        raise ConnectionError(
+            f"Connection error for {subject} ({type(e).__name__})"
+        ) from None

pathapi/http/_follow.py

@@ -0,0 +1,230 @@
+"""What a backend does above a transport: credentials, redirects and the
+errors it raises for a redirect it will not follow."""
+
+from __future__ import annotations
+
+import base64 as _base64
+import errno as _errno
+import re as _re
+import urllib.parse as _urlparse
+
+from ._headers import check_url, drop_header, peer_text
+from ._response import HTTPResponse
+
+MAX_REDIRECTS = 30
+"""Most redirects a `GET`, `HEAD` or `OPTIONS` follows."""
+
+FOLLOWED_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
+"""Methods whose redirects are followed. Every other method goes through
+`send_once()`: following would turn a state-changing one into a different
+request, and a PROPFIND into a GET or one without its body."""
+
+RESENT_STATUSES = {"PROPFIND": (301, 302, 303, 307, 308)}
+"""The redirect statuses after which a method is sent again, same method and
+body, to a Location on the same scheme, host and port. PROPFIND only reads,
+so it is repeated for every redirect; any other method only for 307 and 308,
+which are defined to keep the method and the body."""
+
+_REDIRECTS = (301, 302, 303, 307, 308)
+_DEFAULT_PORTS = {"http": 80, "https": 443}
+_RE_URL_SCHEME = _re.compile(r"[A-Za-z][A-Za-z0-9+.-]*://")
+_DRAIN = 64 * 1024
+
+
+def credential(text: str) -> "str | bytes":
+    """The octets a percent-encoded userinfo part stands for (RFC 3986 2.1),
+    as `str` when they are ASCII and as `bytes` otherwise."""
+    octets = _urlparse.unquote_to_bytes(text)
+    return octets.decode("ascii") if octets.isascii() else octets
+
+
+def split_userinfo(url: str) -> "tuple[str, tuple[str|bytes, str|bytes] | None]":
+    """Split the userinfo out of an absolute URL: `(url_without_userinfo,
+    (user, password) | None)`, both parts percent-decoded to the octets they
+    stand for. Credentials inside a request URL would end up in
+    `HTTPResponse.url`, in error text and in redirect handling; they are sent
+    in an `Authorization` header instead."""
+    match = _RE_URL_SCHEME.match(url)
+    if not match:
+        return url, None
+    start = match.end()
+    end = len(url)
+    for sep in "/?#":
+        index = url.find(sep, start)
+        if index != -1 and index < end:
+            end = index
+    userinfo, at, hostport = url[start:end].rpartition("@")
+    if not at:
+        return url, None
+    user, _, password = userinfo.partition(":")
+    auth = (credential(user), credential(password))
+    return url[:start] + hostport + url[end:], (auth if any(auth) else None)
+
+
+def basic_authorization(auth: "tuple[str | bytes, str | bytes]") -> str:
+    """The `Authorization` value for Basic credentials (RFC 7617)."""
+    user, password = (p.encode("utf-8") if isinstance(p, str) else p for p in auth)
+    return "Basic " + _base64.b64encode(user + b":" + password).decode("ascii")
+
+
+def origin(url: str) -> "tuple[str, str, int | None]":
+    """The scheme, host and port a URL names."""
+    split = _urlparse.urlsplit(url)
+    scheme = split.scheme.lower()
+    try:
+        port = split.port
+    except ValueError:
+        port = -1
+    # `hostname` is lower case already; a host name is case-insensitive.
+    return scheme, (split.hostname or "").lower(), port or _DEFAULT_PORTS.get(scheme)
+
+
+def shown(url: str) -> str:
+    """`url` as an error message may print it: no userinfo, query or fragment,
+    which can carry a credential."""
+    try:
+        split = _urlparse.urlsplit(url)
+        netloc = split.netloc.rpartition("@")[2]
+        return _urlparse.urlunsplit((split.scheme, netloc, split.path, "", ""))
+    except ValueError:
+        return "(unreadable URL)"
+
+
+def is_redirect(resp: HTTPResponse) -> bool:
+    return resp.status in _REDIRECTS and "Location" in resp.headers
+
+
+def redirect_location(url: str, resp: HTTPResponse) -> "str | None":
+    """`resp`'s `Location` as an absolute `http:`/`https:` URL without
+    userinfo, or `None` when there is none or it cannot be used."""
+    location = resp.headers.get("Location")
+    if not location:
+        return None
+    try:
+        # http.client decodes header bytes as Latin-1; servers send UTF-8.
+        location = location.encode("latin1").decode("utf-8")
+    except (UnicodeEncodeError, UnicodeDecodeError):
+        pass
+    if any(ord(c) < 0x20 or ord(c) == 0x7F for c in location):
+        return None
+    try:
+        absolute = split_userinfo(_urlparse.urljoin(url, location))[0]
+        # What a transport would refuse is not a place to go.
+        check_url(absolute)
+        return absolute
+    except ValueError:
+        return None
+
+
+def refused_redirect(method: str, url: str, resp: HTTPResponse, location, reason):
+    status = f"HTTP {resp.status} {peer_text(resp.reason or '')}".rstrip()
+    where = peer_text(shown(location), 200) if location else "(none)"
+    return OSError(
+        _errno.EIO,
+        f"{status} for {method} {shown(url)}: redirect to {where} "
+        f"not followed ({reason})",
+    )
+
+
+def _discard(resp: HTTPResponse) -> None:
+    """Gives up the rest of a body nobody wants, reading a small one so the
+    connection can serve the next request."""
+    try:
+        resp.read(_DRAIN)
+    except OSError:
+        pass
+    finally:
+        resp.close()
+
+
+def can_resend_body(body) -> bool:
+    return body is None or isinstance(body, (bytes, bytearray, memoryview))
+
+
+def send_following(
+    transport,
+    method: str,
+    url: str,
+    *,
+    headers: "dict[str, str]",
+    timeout,
+    stream: bool,
+    first: "str | None" = None,
+) -> HTTPResponse:
+    """Send a request that carries no body and follow its redirects, up to
+    `MAX_REDIRECTS`. `Authorization` goes to the origin of `first` (the URL
+    the caller asked for; `url` when it is not given) and nowhere else: a
+    request to another scheme, host or port drops it, and for every later
+    hop. A `303` becomes a `GET` (unless the request was a `HEAD`)."""
+    home = origin(first or url)
+    current = url
+    hops = 0
+    while True:
+        if origin(current) != home:
+            headers = drop_header(headers, "Authorization")
+        resp = transport.send(
+            method, current, headers=headers, body=None, timeout=timeout, stream=stream
+        )
+        if not is_redirect(resp):
+            return resp
+        location = redirect_location(current, resp)
+        if location is None:
+            _discard(resp)
+            raise refused_redirect(method, current, resp, None, "no usable Location")
+        if hops >= MAX_REDIRECTS:
+            _discard(resp)
+            raise OSError(
+                _errno.EIO,
+                f"more than {MAX_REDIRECTS} redirects from {shown(url)}",
+            )
+        _discard(resp)
+        if resp.status == 303 and method != "HEAD":
+            method = "GET"
+        current = location
+        hops += 1
+
+
+def send_once(
+    transport,
+    method: str,
+    url: str,
+    *,
+    headers: "dict[str, str]",
+    body,
+    timeout,
+    stream: bool,
+) -> HTTPResponse:
+    """Send a request that must not be redirected the way a browser does: a
+    redirect with a status in `RESENT_STATUSES` to the same scheme, host and
+    port is sent once more with the same method and body, and any other 3xx
+    raises `OSError(EIO)` naming the status and the `Location`."""
+    resp = transport.send(
+        method, url, headers=headers, body=body, timeout=timeout, stream=stream
+    )
+    if not 300 <= resp.status < 400:
+        return resp
+    location = redirect_location(url, resp)
+    resent = RESENT_STATUSES.get(method.upper(), (307, 308))
+    if resp.status not in resent:
+        reason = (
+            "only "
+            + ", ".join(map(str, resent[:-1]))
+            + f" and {resent[-1]} send the request again"
+        )
+    elif location is None:
+        reason = "no usable Location"
+    elif origin(location) != origin(url):
+        reason = "it leaves the scheme, host and port of the request"
+    elif not can_resend_body(body):
+        reason = "the request body cannot be sent twice"
+    else:
+        _discard(resp)
+        resp = transport.send(
+            method, location, headers=headers, body=body, timeout=timeout, stream=stream
+        )
+        if not 300 <= resp.status < 400:
+            return resp
+        url, location = location, redirect_location(location, resp)
+        reason = "a second redirect"
+    _discard(resp)
+    raise refused_redirect(method, url, resp, location, reason)