xtr-http-client 4.0.2__py3-none-any.whl

xtr_http_client/internal/mock_head.py

@@ -0,0 +1,144 @@
+"""The head of a scripted answer, read out of the info a script was written with.
+
+A scripted response is written as a body and a mapping of what the transfer is
+to claim about itself. The status line, the protocol version and the headers
+are all in that mapping, under one entry and in any of three shapes, so reading
+them out is one decision made here rather than at each place that needs one of
+them.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable, Mapping, Sequence
+from typing import TYPE_CHECKING, Final, cast
+
+from xtr_http_message import Response
+
+if TYPE_CHECKING:
+    from xtr_http_message_contracts import ResponseInterface
+
+__all__ = ["header_lines", "response_header_lines", "scripted_message"]
+
+# "HTTP/1.1 404 Not Found": what a hop begins with, and the one line in the
+# headers that is not a header.
+_STATUS_LINE: Final = re.compile(r"^HTTP/(\d+(?:\.\d+)?) (\d{3})(?: (.*))?$")
+
+_DEFAULT_STATUS: Final = 200
+_DEFAULT_VERSION: Final = "1.1"
+
+
+def scripted_message(info: Mapping[str, object], length: int | None) -> Response:
+    """The status line and headers a scripted answer arrives with.
+
+    Args:
+        info: What the script says the transfer claims about itself.
+            ``http_code`` is the status, and ``response_headers`` the headers —
+            which a status line among them overrides, since a line the script
+            wrote out says the version and the reason phrase as well.
+        length: The real length of the body, when the body is one that has a
+            length. ``Content-Length`` is then set to it, so a script that
+            claims a length its body does not have cannot leave a reader
+            waiting for bytes that are never coming. ``None`` — a body
+            produced piece by piece — leaves whatever was claimed.
+
+    Returns:
+        The message the answer is read from.
+    """
+    status = _status(info.get("http_code"))
+    version = _DEFAULT_VERSION
+    reason: str | None = None
+    headers: dict[str, list[str]] = {}
+
+    for line in response_header_lines(info.get("response_headers")):
+        hop = _STATUS_LINE.match(line)
+        if hop is not None:
+            # No phrase at all (`None`) asked for the recommended one; an
+            # empty phrase after the space asked for none, and gets none.
+            version, status, reason = hop[1], int(hop[2]), hop[3]
+            # A new hop answers for itself: the headers before it were the
+            # previous one's.
+            headers.clear()
+            continue
+
+        name, separator, value = line.partition(":")
+        if separator:
+            headers.setdefault(_first_cased(headers, name), []).append(value.lstrip())
+
+    if length is not None:
+        headers[_first_cased(headers, "Content-Length")] = [str(length)]
+
+    return Response(status, headers, None, version, reason)
+
+
+def response_header_lines(value: object) -> list[str]:
+    """Headers written in any of the three shapes a script may write them, as lines.
+
+    A mapping is the shape a header is named in; a sequence of lines is the
+    shape a transfer reports, and the only one that can carry a status line;
+    a single string is one of those lines on its own.
+
+    Args:
+        value: What the script gave for ``response_headers``.
+
+    Returns:
+        One line per value, each ``"Name: value"`` or a status line.
+    """
+    if isinstance(value, str):
+        return [value]
+    if isinstance(value, Mapping):
+        mapping = cast("Mapping[object, object]", value)
+
+        return [f"{name}: {one}" for name, values in mapping.items() for one in _each(values)]
+    if isinstance(value, Iterable):
+        return [str(line) for line in value]
+
+    return []
+
+
+def header_lines(message: ResponseInterface) -> list[str]:
+    """The raw status and header lines a message would have arrived as.
+
+    Args:
+        message: The message the answer is read from.
+
+    Returns:
+        The status line, then one line per header value.
+    """
+    status_line = (
+        f"HTTP/{message.get_protocol_version()} "
+        f"{message.get_status_code()} {message.get_reason_phrase()}"
+    )
+
+    return [
+        status_line,
+        *(f"{name}: {value}" for name, values in message.get_headers().items() for value in values),
+    ]
+
+
+def _each(values: object) -> list[str]:
+    """Every value given for one header name."""
+    if isinstance(values, str):
+        return [values]
+    if isinstance(values, Sequence):
+        return [str(one) for one in values]
+
+    return [str(values)]
+
+
+def _status(value: object) -> int:
+    """The status the script claims, or the one an answer has when nobody said."""
+    if isinstance(value, bool) or not isinstance(value, int):
+        return _DEFAULT_STATUS
+
+    return value
+
+
+def _first_cased(headers: Mapping[str, list[str]], name: str) -> str:
+    """The case a header name was first seen in, so a repeat does not split it."""
+    wanted = name.lower()
+    for seen in headers:
+        if seen.lower() == wanted:
+            return seen
+
+    return name

xtr_http_client/internal/mock_info.py

@@ -0,0 +1,86 @@
+"""What a scripted transfer knows about itself, once a client has issued it."""
+
+from __future__ import annotations
+
+import time
+from typing import TYPE_CHECKING, Final
+
+from .mock_head import header_lines
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from xtr_http_message_contracts import ResponseInterface
+
+__all__ = ["issued_info"]
+
+# The entries the client fills in, whatever the script says: a script cannot
+# know the method, the URL or the options it is going to answer, and the rest
+# are what sending it made true. Everything else a script wrote is carried
+# through untouched, which is how a test puts an entry of its own on a
+# response.
+_FILLED_IN: Final = frozenset(
+    {
+        "canceled",
+        "error",
+        "http_code",
+        "http_method",
+        "redirect_count",
+        "redirect_url",
+        "response_headers",
+        "start_time",
+        "total_time",
+        "url",
+        "user_data",
+    }
+)
+
+
+def issued_info(
+    script: Mapping[str, object],
+    message: ResponseInterface,
+    method: str,
+    url: str,
+    options: Mapping[str, object],
+) -> dict[str, object]:
+    """The record of a scripted transfer, as it stands the moment it is handed back.
+
+    Args:
+        script: What the response was written with, whose entries outside the
+            ones filled in here are kept as they were given.
+        message: The status line and headers the answer arrives with.
+        method: The method the request was made with.
+        url: The URL it was made to, resolved.
+        options: The options it was made with, merged.
+
+    Returns:
+        Every entry a caller can read off the response.
+    """
+    given_start = _reading(script.get("start_time"))
+    start_time = time.time() if given_start is None else given_start
+    lines = header_lines(message)
+    total_time = _reading(script.get("total_time"))
+
+    return {
+        **{name: value for name, value in script.items() if name not in _FILLED_IN},
+        "canceled": False,
+        "error": None,
+        "http_code": message.get_status_code(),
+        "http_method": method,
+        # Nothing is followed: a script answers the request that was made.
+        "redirect_count": 0,
+        "redirect_url": None,
+        "response_headers": lines,
+        "start_time": start_time,
+        "total_time": total_time if total_time is not None else time.time() - start_time,
+        "url": url,
+        "user_data": options.get("user_data"),
+    }
+
+
+def _reading(value: object) -> float | None:
+    """A clock reading the script gave, or ``None`` when it gave none."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+
+    return float(value)

xtr_http_client/internal/option_merge.py

@@ -0,0 +1,108 @@
+"""What a request asked for, over what its client always asks for."""
+
+from __future__ import annotations
+
+import difflib
+from typing import TYPE_CHECKING, cast
+
+from xtr_http_client.exception import InvalidArgumentError
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from xtr_http_client_contracts import RequestOptions
+
+__all__ = ["as_options", "merge_default_options", "validate_options"]
+
+
+def as_options(options: Mapping[str, object]) -> RequestOptions:
+    """``options`` read as the typed options, for handing on as keyword arguments.
+
+    A client that builds options name by name — adding one, dropping another —
+    holds them as a plain mapping, which is the same thing the contract types
+    key by key. This says so once, rather than at every call that passes them
+    on.
+    """
+    return cast("RequestOptions", options)
+
+
+def merge_default_options(
+    client_name: str,
+    options: Mapping[str, object],
+    default_options: Mapping[str, object],
+) -> dict[str, object]:
+    """Merge what a request asked for over what its client always asks for.
+
+    An option the request named wins whole — there is no merging *inside* an
+    option, because half a timeout or half a body means nothing. ``extra`` and
+    ``resolve`` are the exceptions, and merge by key: an entry the defaults
+    carry that the request did not mention is kept, which is what makes a
+    client-wide setting work, while one it did mention is the request's.
+
+    Args:
+        client_name: The client the options are for, named in the message when
+            one of them is not supported.
+        options: What the request asked for.
+        default_options: What the client always asks for, and the list of what
+            it understands at all.
+
+    Returns:
+        Every option, the request's over the client's.
+
+    Raises:
+        InvalidArgumentError: When an option is not one this client knows.
+    """
+    merged: dict[str, object] = dict(options)
+    for name, value in default_options.items():
+        _ = merged.setdefault(name, value)
+
+    if "extra" in default_options:
+        merged["extra"] = _filled(merged.get("extra"), default_options["extra"])
+    if default_options.get("resolve"):
+        merged["resolve"] = _filled(merged.get("resolve"), default_options["resolve"])
+
+    if default_options:
+        validate_options(client_name, merged, default_options)
+
+    return merged
+
+
+def validate_options(
+    client_name: str,
+    options: Mapping[str, object],
+    default_options: Mapping[str, object],
+) -> None:
+    """Refuse an option this client does not know, naming the closest it does.
+
+    Refused rather than ignored, since a misspelled option that is quietly
+    dropped is a setting that silently never applied.
+
+    Args:
+        client_name: The client the options are for, named in the message.
+        options: The names to check.
+        default_options: The list of what the client understands at all.
+
+    Raises:
+        InvalidArgumentError: When an option is not one this client knows.
+    """
+    for name in options:
+        if name in default_options:
+            continue
+
+        close = difflib.get_close_matches(
+            name, list(default_options), n=len(default_options), cutoff=0.6
+        )
+        containing = [known for known in default_options if name in known]
+        alternatives = list(dict.fromkeys([*close, *containing]))
+        did_you_mean = '", "'.join(alternatives or list(default_options))
+        raise InvalidArgumentError(
+            f'Unsupported option "{name}" passed to "{client_name}", did you mean "{did_you_mean}"?'
+        )
+
+
+def _filled(given: object, defaults: object) -> dict[str, object]:
+    filled: dict[str, object] = dict(cast("Mapping[str, object]", given or {}))
+    for name, value in cast("Mapping[str, object]", defaults).items():
+        _ = filled.setdefault(name, value)
+
+    return filled

xtr_http_client/internal/response_body.py

@@ -0,0 +1,270 @@
+"""A message body read as the transfer produces it."""
+
+from __future__ import annotations
+
+import os
+from tempfile import SpooledTemporaryFile
+from typing import TYPE_CHECKING, Final, Protocol, cast, final, runtime_checkable
+
+from xtr_http_message import Stream
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+    from typing import BinaryIO
+
+    from xtr_http_message_contracts import StreamInterface
+
+__all__ = ["ResponseBody"]
+
+_READ_SIZE: Final = 65536
+
+# Where a body stops being held in memory and starts being held in a file:
+# enough that an ordinary response never reaches the disk, little enough that
+# a download nobody sized cannot exhaust what is left of memory.
+_SPILL_AT: Final = 2 * 1024 * 1024
+
+
+@runtime_checkable
+class _ClosableInterface(Protocol):
+    """A source with something of its own to let go of."""
+
+    async def aclose(self) -> None:
+        """Let go of whatever the source was holding open."""
+        ...
+
+
+@final
+class ResponseBody:
+    """The body of a response, taken off the transfer as it is asked for.
+
+    A message body is a cursor over bytes; a body still arriving has no bytes
+    to be a cursor over yet. So a read that finds nothing waiting takes the
+    next piece from the transfer, one piece at a time, until bytes turn up or
+    the transfer is over — which the source says by ending. Nothing is fetched
+    that nobody asked for, which is what makes a body the caller never reads
+    cost nothing.
+
+    Everything that arrives is written into a stream of its own, so the body
+    can be read again — and is then seekable, once complete, because only then
+    is there a whole body to seek in. That stream holds the first two
+    mebibytes in memory and spills the rest to a file, so a large download is
+    re-readable without being resident.
+
+    Writing is not part of it: the bytes come from the far end.
+    """
+
+    __slots__ = ("_closed", "_complete", "_kept", "_received", "_source")
+
+    def __init__(self, source: AsyncIterator[bytes]) -> None:
+        """Hold onto the transfer this body comes out of.
+
+        Args:
+            source: The pieces of the body, in the order they arrive. Ending
+                is how it says the transfer is complete. Whatever else it
+                raises is left alone: what a broken transfer means for a
+                response is the response's to decide, not the body's.
+        """
+        self._source = source
+        self._kept: StreamInterface | None = None
+        self._received = 0
+        self._complete = False
+        self._closed = False
+
+    @property
+    def complete(self) -> bool:
+        """Whether the whole body has arrived."""
+        return self._complete
+
+    @property
+    def received(self) -> int:
+        """How many bytes have arrived so far."""
+        return self._received
+
+    async def close(self) -> None:
+        """Let go of the body, the transfer behind it, and what it was kept in.
+
+        Nothing can be read from it afterwards.
+        """
+        self._closed = True
+
+        if isinstance(self._source, _ClosableInterface):
+            await self._source.aclose()
+
+        if self._kept is not None:
+            await self._kept.close()
+
+    def detach(self) -> BinaryIO | None:
+        """Return nothing: what a body is kept in is its own, not a caller's to take over."""
+        return None
+
+    def get_size(self) -> int | None:
+        """Return the size of the body, which is known once it is complete."""
+        if not self._complete:
+            return None
+        if self._kept is None:
+            return self._received
+        return self._kept.get_size()
+
+    def tell(self) -> int:
+        """Return how far into the body reading has got."""
+        if self._kept is None:
+            return 0
+        return self._kept.tell()
+
+    def eof(self) -> bool:
+        """Tell whether reading has reached the end of a body that has all arrived."""
+        return self._complete and self.tell() >= self._received
+
+    def is_seekable(self) -> bool:
+        """Tell whether the cursor can be moved.
+
+        Only once the whole body has arrived: before that there is no whole
+        body to move about in.
+        """
+        return not self._closed and self._complete
+
+    async def seek(self, offset: int, whence: int = os.SEEK_SET) -> None:
+        """Move the cursor to a position in the body.
+
+        Raises:
+            ValueError: When ``whence`` is not one of the three the standard
+                library defines.
+            RuntimeError: When this body cannot be seeked in, or the position
+                asked for is before its first byte.
+        """
+        if not self.is_seekable():
+            raise RuntimeError(
+                "The response body cannot be rewound: it is read as the transfer produces it."
+            )
+
+        origins = {os.SEEK_SET: 0, os.SEEK_CUR: self.tell(), os.SEEK_END: self._received}
+        if whence not in origins:
+            raise ValueError(f"Invalid whence: {whence}.")
+
+        target = origins[whence] + offset
+        if target < 0:
+            raise RuntimeError(f"Cannot seek to {target}: that is before the body starts.")
+
+        if self._kept is not None:
+            await self._kept.seek(target)
+
+    async def rewind(self) -> None:
+        """Move the cursor back to the first byte of the body.
+
+        Raises:
+            RuntimeError: When this body cannot be seeked in.
+        """
+        await self.seek(0)
+
+    def is_writable(self) -> bool:
+        """Say no: the bytes of a response come from the far end."""
+        return False
+
+    async def write(self, data: bytes) -> int:
+        """Refuse to write.
+
+        Raises:
+            RuntimeError: Always.
+        """
+        del data
+        raise RuntimeError("A response body cannot be written to.")
+
+    def is_readable(self) -> bool:
+        """Tell whether the body can still be read."""
+        return not self._closed
+
+    async def read(self, length: int) -> bytes:
+        """Read up to ``length`` bytes from the cursor.
+
+        Takes pieces off the transfer until bytes turn up past the cursor or
+        the transfer is over, then hands over what is there — which may be
+        less than was asked for, because a transfer arrives in pieces and
+        waiting for the rest would hold up a reader that has something to do
+        with this piece.
+
+        Returns:
+            What was read, or empty bytes once the body is exhausted.
+
+        Raises:
+            RuntimeError: When the body has been closed.
+        """
+        if self._closed:
+            raise RuntimeError("The response body is closed.")
+        if length <= 0:
+            return b""
+
+        while self._waiting() == 0 and not self._complete:
+            await self._receive()
+
+        if self._kept is None:
+            return b""
+        return await self._kept.read(length)
+
+    async def get_contents(self) -> bytes:
+        """Read the whole of the body, rewinding first when that is possible.
+
+        Returns:
+            Every byte left in the body.
+
+        Raises:
+            RuntimeError: When the body has been closed.
+        """
+        if self.is_seekable():
+            await self.rewind()
+
+        pieces: list[bytes] = []
+        while True:
+            piece = await self.read(_READ_SIZE)
+            if not piece:
+                return b"".join(pieces)
+            pieces.append(piece)
+
+    def get_metadata(self, key: str | None = None) -> object:
+        """Return what this body knows about itself, or one entry of it."""
+        metadata: dict[str, object] = {
+            "mode": "rb",
+            "seekable": self.is_seekable(),
+            "eof": self.eof(),
+        }
+        if key is None:
+            return metadata
+        return metadata.get(key)
+
+    async def _receive(self) -> None:
+        """Take the next piece off the transfer, or learn that there is none."""
+        try:
+            piece = await anext(self._source)
+        except StopAsyncIteration:
+            self._complete = True
+            return
+        await self._arrived(piece)
+
+    async def _arrived(self, piece: bytes) -> None:
+        """Count a piece in, and keep it where a reader can still reach it."""
+        self._received += len(piece)
+
+        # The kept stream's cursor *is* the read cursor, so an arriving piece
+        # is appended at the end and the cursor put back where the reader left
+        # it.
+        kept = self._keep()
+        reading_at = kept.tell()
+        await kept.seek(0, os.SEEK_END)
+        _ = await kept.write(piece)
+        await kept.seek(reading_at)
+
+    def _keep(self) -> StreamInterface:
+        """Return the stream the body is held in, opening it on first use."""
+        if self._kept is None:
+            # Held open past this call on purpose: `close` is what lets go of
+            # it. And it is a binary file object in every way a stream uses
+            # one — only typed as the wider `IO[bytes]`, whose `write` takes
+            # the narrower `bytes` a body ever passes.
+            spooled = SpooledTemporaryFile(max_size=_SPILL_AT)  # noqa: SIM115
+            self._kept = Stream.create(cast("BinaryIO", cast("object", spooled)))
+        return self._kept
+
+    def _waiting(self) -> int:
+        """How many bytes have arrived that the cursor has not reached."""
+        if self._kept is None:
+            return 0
+        return max(0, self._received - self._kept.tell())