pathapi 0.1.0__py3-none-any.whl

pathapi/http/_headers.py

@@ -0,0 +1,233 @@
+"""Headers and the checks a request passes before anything is sent."""
+
+from __future__ import annotations
+
+import collections.abc as _abc
+import re as _re
+import typing as _ty
+import urllib.parse as _urlparse
+
+_TOKEN = _re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
+_CONTROL = _re.compile(r"[\x00-\x08\x0a-\x1f\x7f]")
+_URL_CONTROL = _re.compile(r"[\x00-\x1f\x7f]")
+_NOT_PRINTABLE = _re.compile(r"[^\x21-\x7e]")
+
+_DEFAULT_PORTS = {"http": 80, "https": 443}
+
+
+class Headers(_abc.Mapping):
+    """The headers of a response: a read-only mapping whose names compare
+    without regard to case. A name sent more than once reads as its values
+    joined with `", "` (RFC 9110 5.3); `get_all()` returns them separately."""
+
+    __slots__ = ("_pairs", "_joined", "_spelling")
+
+    def __init__(self, pairs: "_ty.Iterable[tuple[str, str]] | _abc.Mapping" = ()):
+        if isinstance(pairs, _abc.Mapping):
+            pairs = pairs.items()
+        self._pairs = tuple((str(name), str(value)) for name, value in pairs)
+        grouped: "dict[str, list[str]]" = {}
+        self._spelling: "dict[str, str]" = {}
+        for name, value in self._pairs:
+            key = name.lower()
+            grouped.setdefault(key, []).append(value)
+            self._spelling.setdefault(key, name)
+        self._joined = {key: ", ".join(values) for key, values in grouped.items()}
+
+    def __getitem__(self, name: str) -> str:
+        return self._joined[name.lower()]
+
+    def __iter__(self) -> "_ty.Iterator[str]":
+        return iter(self._spelling.values())
+
+    def __len__(self) -> int:
+        return len(self._joined)
+
+    def __contains__(self, name: object) -> bool:
+        return isinstance(name, str) and name.lower() in self._joined
+
+    def get_all(self, name: str) -> "list[str]":
+        """Every value sent for `name`, in order."""
+        key = name.lower()
+        return [value for field, value in self._pairs if field.lower() == key]
+
+    def __repr__(self) -> str:
+        # Values are left out: a reply can carry cookies and tokens.
+        return f"Headers({list(self._spelling.values())!r})"
+
+
+def peer_text(text: object, limit: int = 120) -> str:
+    """`text` that came from the other end (a reason phrase, a `Location`, a
+    header value) as an error message may show it: characters a terminal or a
+    log would act on (controls, line and bidirectional separators) escaped, and
+    cut to `limit` characters."""
+    text = str(text)
+    cut = len(text) > limit
+    shown = "".join(
+        char if char.isprintable() else char.encode("unicode_escape").decode("ascii")
+        for char in text[:limit]
+    )
+    return shown + "..." if cut else shown
+
+
+def has_header(headers: "_abc.Mapping[str, str]", name: str) -> bool:
+    key = name.lower()
+    return any(field.lower() == key for field in headers)
+
+
+def wants_decoding(pairs: "_ty.Iterable[tuple[str, str]]") -> bool:
+    """Whether a response is decoded for a request with these headers: unless
+    the request names a content coding of its own in `Accept-Encoding`
+    (`identity` is not one, so a server that encodes anyway is still decoded)."""
+    for name, value in pairs:
+        if name.lower() == "accept-encoding":
+            tokens = {part.split(";")[0].strip().lower() for part in value.split(",")}
+            return tokens <= {"", "identity"}
+    return True
+
+
+def drop_header(headers: "_abc.Mapping[str, str]", name: str) -> "dict[str, str]":
+    """`headers` as a new dict without `name`, whatever its case."""
+    key = name.lower()
+    return {field: value for field, value in headers.items() if field.lower() != key}
+
+
+def merge_headers(*layers: "_abc.Mapping[str, str] | None") -> "dict[str, str]":
+    """The layers laid over one another, a later one winning on a name whatever
+    its case; a value of `None` removes the name."""
+    merged: "dict[str, tuple[str, str]]" = {}
+    for layer in layers:
+        for name, value in (layer or {}).items():
+            if value is None:
+                merged.pop(name.lower(), None)
+            else:
+                merged[name.lower()] = (name, value)
+    return dict(merged.values())
+
+
+def check_timeout(timeout) -> "tuple[float | None, float | None]":
+    """`timeout` (a number, `None`, or a `(connect, read)` pair) as that pair;
+    `ValueError` for anything else."""
+    if timeout is None:
+        return None, None
+    if isinstance(timeout, tuple):
+        if len(timeout) != 2:
+            raise ValueError("a timeout is a number, None or a (connect, read) pair")
+        connect, read = timeout
+    else:
+        connect = read = timeout
+    for value in (connect, read):
+        if value is not None and (
+            isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0
+        ):
+            raise ValueError("a timeout is a positive number of seconds, or None")
+    return connect, read
+
+
+def check_method(method: str) -> str:
+    if not isinstance(method, str) or not _TOKEN.fullmatch(method):
+        raise ValueError("the request method is not an HTTP token")
+    return method.upper()
+
+
+def check_body(body) -> bool:
+    """Whether `body` (`None`, bytes, a binary file or an iterable of bytes)
+    can be sent twice; `TypeError` for text or anything else."""
+    if body is None or isinstance(body, (bytes, bytearray, memoryview)):
+        return True
+    if isinstance(body, str):
+        raise TypeError("the request body is bytes, not text")
+    if hasattr(body, "read"):
+        return False
+    if isinstance(body, (list, tuple)):
+        return all(isinstance(piece, (bytes, bytearray)) for piece in body)
+    if hasattr(body, "__iter__"):
+        return False
+    raise TypeError("the request body is bytes, a file or an iterable of bytes")
+
+
+def check_headers(headers: "_abc.Mapping[str, str] | None") -> "list[tuple[str, str]]":
+    """`headers` as `(name, value)` text pairs, or `ValueError` for a name that
+    is not an HTTP token or a value with a control character, a line break
+    included. The refused text is not quoted: it may be a credential."""
+    pairs = []
+    for name, value in (headers or {}).items():
+        if not isinstance(name, str) or not _TOKEN.fullmatch(name):
+            raise ValueError("a request header name is not an HTTP token")
+        if isinstance(value, bytes):
+            value = value.decode("latin-1")
+        elif isinstance(value, int) and not isinstance(value, bool):
+            value = str(value)
+        if not isinstance(value, str):
+            raise ValueError(f"the value of the {name} header is not text")
+        if _CONTROL.search(value):
+            raise ValueError(f"the value of the {name} header has a control character")
+        try:
+            value.encode("latin-1")
+        except UnicodeEncodeError:
+            raise ValueError(f"the value of the {name} header is not Latin-1") from None
+        pairs.append((name, value))
+    return pairs
+
+
+class Target(_ty.NamedTuple):
+    """Where a request goes and how it names itself on the wire."""
+
+    scheme: str
+    host: str  # IDNA, without brackets
+    port: int
+    path: str  # the origin-form request target, percent-encoded
+    authority: str  # the Host header value
+
+
+def _percent(match: "_re.Match") -> str:
+    return "".join(
+        f"%{byte:02X}" for byte in match.group().encode("utf-8", "surrogateescape")
+    )
+
+
+def bracket(host: str) -> str:
+    return f"[{host}]" if ":" in host else host
+
+
+def check_url(url: str) -> Target:
+    """The parts of an `http:`/`https:` URL a request is made of. A control
+    character (CR and LF included), credentials in the URL, another scheme or
+    a missing host are `ValueError`; the URL is not quoted in the message."""
+    if not isinstance(url, str):
+        raise ValueError("the request URL is not text")
+    if _URL_CONTROL.search(url):
+        raise ValueError("the request URL has a control character")
+    try:
+        parts = _urlparse.urlsplit(url)
+        port = parts.port
+    except ValueError:
+        raise ValueError("the request URL cannot be parsed") from None
+    scheme = parts.scheme.lower()
+    if scheme not in _DEFAULT_PORTS:
+        raise ValueError(f"the request URL has scheme {scheme!r}, not http or https")
+    if "@" in parts.netloc:
+        raise ValueError(
+            "the request URL carries credentials; send them in an Authorization header"
+        )
+    # The connection pool, the Host header, SNI and the proxy rules key on this
+    # name, which is case-insensitive: `hostname` is lower case already.
+    host = (parts.hostname or "").lower()
+    if not host:
+        raise ValueError("the request URL has no host")
+    if not host.isascii():
+        try:
+            host = host.encode("idna").decode("ascii")
+        except UnicodeError:
+            raise ValueError(
+                "the host of the request URL is not a valid name"
+            ) from None
+    port = port or _DEFAULT_PORTS[scheme]
+    path = parts.path or "/"
+    if parts.query:
+        path += "?" + parts.query
+    path = _NOT_PRINTABLE.sub(_percent, path)
+    authority = bracket(host)
+    if port != _DEFAULT_PORTS[scheme]:
+        authority += f":{port}"
+    return Target(scheme, host, port, path, authority)

pathapi/http/_pool.py

@@ -0,0 +1,169 @@
+"""The idle connections of a transport: one per thread and server."""
+
+from __future__ import annotations
+
+import collections as _collections
+import itertools as _itertools
+import socket as _socket
+import threading as _threading
+import typing as _ty
+import weakref as _weakref
+
+MAX_IDLE_PER_THREAD = 8
+"""Most idle connections one thread keeps open; opening another closes the one
+used longest ago."""
+
+
+class Connection:
+    """An open `http.client.HTTPConnection` and what the pool knows of it."""
+
+    __slots__ = ("http", "key", "scope", "released_at", "generation", "__weakref__")
+
+    def __init__(self, http, key: tuple, scope: int, generation: int):
+        self.http = http
+        self.key = key
+        self.scope = scope
+        self.generation = generation
+        self.released_at = 0.0
+
+    def close(self, interrupt: bool = False) -> None:
+        """Closes the socket. `interrupt` also shuts it down first, which wakes
+        a thread blocked on it."""
+        http = self.http
+        sock = getattr(http, "sock", None)
+        try:
+            if interrupt and sock is not None:
+                try:
+                    sock.shutdown(_socket.SHUT_RDWR)
+                except OSError:
+                    pass
+        finally:
+            try:
+                http.close()
+            except Exception:
+                pass
+
+
+_SCOPE_IDS = _itertools.count(1)
+_THREAD = _threading.local()
+_LIVE_SCOPES: "set[int]" = set()
+_POOLS: "_weakref.WeakSet[Pool]" = _weakref.WeakSet()
+
+
+def _drop_scope(scope: int) -> None:
+    _LIVE_SCOPES.discard(scope)
+    for pool in list(_POOLS):
+        pool.drop_scope(scope)
+
+
+class _ThreadScope:
+    """Held in the thread's own storage, so it is released when the thread ends
+    and takes the thread's idle connections with it."""
+
+    def __init__(self):
+        self.id = next(_SCOPE_IDS)
+        _LIVE_SCOPES.add(self.id)
+        _weakref.finalize(self, _drop_scope, self.id).atexit = False
+
+
+def scope_id() -> int:
+    """The calling thread's scope. Not the thread identifier, which the
+    operating system reuses for a later thread."""
+    scope = getattr(_THREAD, "scope", None)
+    if scope is None:
+        scope = _THREAD.scope = _ThreadScope()
+    return scope.id
+
+
+class Pool:
+    """Idle connections by (thread scope, server), and every connection a
+    transport has open, so `close()` reaches the ones in use too. A connection
+    is taken out of the pool for the length of a response and given back once
+    the body has been read."""
+
+    def __init__(self):
+        self._lock = _threading.Lock()
+        self._idle: "dict[int, _collections.OrderedDict[tuple, Connection]]" = {}
+        self._open: "set[Connection]" = set()
+        self._generation = 0
+        _POOLS.add(self)
+
+    @property
+    def generation(self) -> int:
+        return self._generation
+
+    def take(self, key: tuple, scope: int) -> "Connection | None":
+        """The idle connection of the calling thread to `key`, removed from the
+        pool, or `None`."""
+        with self._lock:
+            return self._idle.get(scope, {}).pop(key, None)
+
+    def register(self, connection: Connection) -> None:
+        with self._lock:
+            self._open.add(connection)
+
+    def give_back(self, connection: Connection, now: float) -> None:
+        """Keeps `connection` idle for the next request of its thread, or
+        closes it when its thread has ended, the transport was closed since it
+        was opened, or the thread already holds one to the same server."""
+        connection.released_at = now
+        evicted = []
+        with self._lock:
+            keep = (
+                connection.scope in _LIVE_SCOPES
+                and connection.generation == self._generation
+            )
+            if keep:
+                idle = self._idle.setdefault(
+                    connection.scope, _collections.OrderedDict()
+                )
+                if connection.key in idle:
+                    keep = False
+                else:
+                    idle[connection.key] = connection
+                    while len(idle) > MAX_IDLE_PER_THREAD:
+                        evicted.append(idle.popitem(last=False)[1])
+            if not keep:
+                self._open.discard(connection)
+                evicted.append(connection)
+            for old in evicted:
+                self._open.discard(old)
+        for old in evicted:
+            old.close()
+
+    def discard(self, connection: Connection) -> None:
+        """Closes a connection that is not to be used again."""
+        with self._lock:
+            self._open.discard(connection)
+        connection.close()
+
+    def drop_scope(self, scope: int) -> None:
+        with self._lock:
+            idle = self._idle.pop(scope, {})
+            for connection in idle.values():
+                self._open.discard(connection)
+        for connection in list(idle.values()):
+            connection.close()
+
+    def close(self) -> None:
+        """Closes every connection, idle or in use. Harmless when called again;
+        a later request connects afresh."""
+        with self._lock:
+            self._generation += 1
+            doomed = list(self._open)
+            self._open.clear()
+            self._idle.clear()
+        for connection in doomed:
+            connection.close(interrupt=True)
+
+
+def close_pool(pool: Pool) -> None:
+    pool.close()
+
+
+def finalize_with(owner: object, pool: Pool) -> "_ty.Any":
+    """Closes `pool` when `owner` is collected, so a transport that was never
+    closed does not leave sockets for the interpreter to report."""
+    handle = _weakref.finalize(owner, close_pool, pool)
+    handle.atexit = False
+    return handle

pathapi/http/_proxies.py

@@ -0,0 +1,142 @@
+"""Which proxies a request goes through, from the environment or a provider."""
+
+from __future__ import annotations
+
+import base64 as _base64
+import errno as _errno
+import ipaddress as _ipaddress
+import typing as _ty
+import urllib.parse as _urlparse
+import urllib.request as _urlrequest
+
+REQUESTS_HINT = "use the requests transport of pathapi-requests for it"
+
+
+class ProxyAddress:
+    """An `http://` proxy: where to connect, and the `Proxy-Authorization`
+    value built from the credentials in its URL. Neither the credentials nor
+    the URL are in `repr()` or in any message."""
+
+    __slots__ = ("host", "port", "authorization")
+
+    def __init__(self, host: str, port: int, authorization: "str | None"):
+        self.host = host
+        self.port = port
+        self.authorization = authorization
+
+    @property
+    def key(self) -> "tuple[str, int]":
+        return (self.host, self.port)
+
+    def __repr__(self) -> str:
+        return f"ProxyAddress({self.host!r}, {self.port})"
+
+
+def _address(proxy: object) -> ProxyAddress:
+    """The address a proxy names: a URL, or any object with a `url`."""
+    url = proxy if isinstance(proxy, str) else getattr(proxy, "url", None)
+    if not isinstance(url, str) or not url.strip():
+        raise ValueError("a proxy is a URL or an object with a url")
+    url = url.strip()
+    parts = _urlparse.urlsplit(url if "://" in url else "http://" + url)
+    scheme = parts.scheme.lower()
+    if scheme != "http":
+        raise OSError(
+            _errno.EPROTONOSUPPORT,
+            f"the built-in HTTP transport has no {scheme!r} proxy support; "
+            + REQUESTS_HINT,
+        )
+    try:
+        host, port = parts.hostname, parts.port
+    except ValueError:
+        raise ValueError("a proxy URL cannot be parsed") from None
+    if not host:
+        raise ValueError("a proxy URL has no host")
+    user, password = parts.username, parts.password
+    if user is None and not isinstance(proxy, str):
+        user = getattr(proxy, "username", None) or None
+        password = getattr(proxy, "password", None) or None
+    authorization = None
+    if user is not None:
+        raw = _urlparse.unquote(user) + ":" + _urlparse.unquote(password or "")
+        authorization = "Basic " + _base64.b64encode(raw.encode("utf-8")).decode(
+            "ascii"
+        )
+    return ProxyAddress(host, port or 80, authorization)
+
+
+def _from_provider(found: object) -> "list[ProxyAddress | None]":
+    if found is None:
+        return [None]
+    if isinstance(found, str) or hasattr(found, "url"):
+        return [_address(found)]
+    try:
+        items = list(found)
+    except TypeError:
+        raise ValueError("a proxy is a URL or an object with a url") from None
+    chain = [None if item is None else _address(item) for item in items]
+    return chain or [None]
+
+
+def _in_no_proxy_network(host: str, no_proxy: str) -> bool:
+    """Whether `host`, an IP literal, is inside a CIDR block (`10.0.0.0/8`,
+    `fd00::/8`) listed in `no_proxy`: the form `requests` honours and
+    `urllib.request.proxy_bypass()` does not."""
+    try:
+        address = _ipaddress.ip_address(host.strip("[]").split("%", 1)[0])
+    except ValueError:
+        return False
+    for entry in no_proxy.split(","):
+        if "/" not in entry:
+            continue
+        try:
+            network = _ipaddress.ip_network(entry.strip(), strict=False)
+        except ValueError:
+            continue
+        if address.version == network.version and address in network:
+            return True
+    return False
+
+
+def _lookup(provider: _ty.Any, url: str, scheme: str, host: str) -> object:
+    """`provider[url]`; for a plain `dict`, the keys `requests` uses for its
+    `proxies` too (`scheme://host`, `scheme`, `all://host`, `all`), so a
+    mapping written for it is not silently ignored."""
+    try:
+        return provider[url]
+    except KeyError:
+        if not isinstance(provider, dict):
+            raise
+    for key in (f"{scheme}://{host}", scheme, f"all://{host}", "all"):
+        if key in provider:
+            return provider[key]
+    raise KeyError(url)
+
+
+def proxies_for(
+    provider: _ty.Any, url: str, scheme: str, host: str
+) -> "list[ProxyAddress | None]":
+    """The proxies to try for `url`, in order; `None` stands for no proxy.
+
+    Without a `provider` the environment decides (`urllib.request.getproxies()`
+    and `proxy_bypass()`, and the CIDR blocks of `NO_PROXY` for an IP-literal
+    host). A provider is indexed with the URL: `KeyError` means no proxy, and
+    the environment is not asked; an empty answer is no proxy too."""
+    if provider is None:
+        chosen = _urlrequest.getproxies()
+        value = chosen.get(scheme) or chosen.get("all")
+        if not value:
+            return [None]
+        try:
+            if _urlrequest.proxy_bypass(host):
+                return [None]
+        except Exception:
+            pass  # a bypass rule that cannot be read does not send it direct
+        if _in_no_proxy_network(host, chosen.get("no", "")):
+            return [None]
+        return [_address(value)]
+    try:
+        found = _lookup(provider, url, scheme, host)
+    except KeyError:
+        return [None]
+    return _from_provider(found)