pyqwest 0.2.0__cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

pyqwest/httpx/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+__all__ = ["AsyncPyqwestTransport", "PyqwestTransport"]
+
+from ._transport import AsyncPyqwestTransport, PyqwestTransport

pyqwest/httpx/_transport.py

@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from typing import TYPE_CHECKING, cast
+
+import httpx
+from h2.errors import ErrorCodes
+from h2.events import StreamReset
+
+from pyqwest import (
+    Headers,
+    HTTPTransport,
+    Request,
+    Response,
+    StreamError,
+    StreamErrorCode,
+    SyncHTTPTransport,
+    SyncRequest,
+    SyncResponse,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Iterator
+
+
+class AsyncPyqwestTransport(httpx.AsyncBaseTransport):
+    """An HTTPX transport implementation that delegates to pyqwest.
+
+    This can be used with any existing code using httpx.AsyncClient, and will enable
+    use of bidirectional streaming and response trailers.
+    """
+
+    _transport: HTTPTransport
+
+    def __init__(self, transport: HTTPTransport) -> None:
+        """Creates a new AsyncPyQwestTransport.
+
+        Args:
+            transport: The pyqwest HTTPTransport to delegate requests to.
+        """
+        self._transport = transport
+
+    async def handle_async_request(
+        self, httpx_request: httpx.Request
+    ) -> httpx.Response:
+        request_headers = convert_headers(httpx_request.headers)
+        request_content = async_request_content(httpx_request.stream)
+        timeout = convert_timeout(httpx_request.extensions)
+
+        try:
+            response = await self._transport.execute(
+                Request(
+                    httpx_request.method,
+                    str(httpx_request.url),
+                    headers=request_headers,
+                    content=request_content,
+                    timeout=timeout,  # pyright: ignore[reportCallIssue]
+                )
+            )
+        except StreamError as e:
+            raise map_stream_error(e) from e
+
+        def get_trailers() -> httpx.Headers:
+            return httpx.Headers(tuple(response.trailers.items()))
+
+        return httpx.Response(
+            status_code=response.status,
+            headers=httpx.Headers(tuple(response.headers.items())),
+            stream=AsyncIteratorByteStream(response),
+            extensions={"get_trailers": get_trailers},
+        )
+
+
+def async_request_content(
+    stream: httpx.AsyncByteStream | httpx.SyncByteStream | httpx.ByteStream,
+) -> bytes | AsyncIterator[bytes]:
+    match stream:
+        case httpx.ByteStream():
+            # Buffered bytes
+            return next(iter(stream))
+        case _:
+            return async_request_content_iter(stream)
+
+
+async def async_request_content_iter(
+    stream: httpx.AsyncByteStream | httpx.SyncByteStream,
+) -> AsyncIterator[bytes]:
+    match stream:
+        case httpx.AsyncByteStream():
+            async with contextlib.aclosing(stream):
+                async for chunk in stream:
+                    yield chunk
+        case httpx.SyncByteStream():
+            with contextlib.closing(stream):
+                stream_iter = iter(stream)
+                while True:
+                    chunk = await asyncio.to_thread(next, stream_iter, None)
+                    if chunk is None:
+                        break
+                    yield chunk
+
+
+class AsyncIteratorByteStream(httpx.AsyncByteStream):
+    def __init__(self, response: Response) -> None:
+        self._response = response
+        self._is_stream_consumed = False
+
+    async def __aiter__(self) -> AsyncIterator[bytes]:
+        if self._is_stream_consumed:
+            raise httpx.StreamConsumed
+        self._is_stream_consumed = True
+        try:
+            async for chunk in self._response.content:
+                yield bytes(chunk)
+        except StreamError as e:
+            raise map_stream_error(e) from e
+
+    async def aclose(self) -> None:
+        await self._response.aclose()
+
+
+class PyqwestTransport(httpx.BaseTransport):
+    """An HTTPX transport implementation that delegates to pyqwest.
+
+    This can be used with any existing code using httpx.Client, and will enable
+    use of bidirectional streaming and response trailers.
+    """
+
+    _transport: SyncHTTPTransport
+
+    def __init__(self, transport: SyncHTTPTransport) -> None:
+        """Creates a new PyQwestTransport.
+
+        Args:
+            transport: The pyqwest HTTPTransport to delegate requests to.
+        """
+        self._transport = transport
+
+    def handle_request(self, httpx_request: httpx.Request) -> httpx.Response:
+        request_headers = convert_headers(httpx_request.headers)
+        request_content = sync_request_content(httpx_request.stream)
+        timeout = convert_timeout(httpx_request.extensions)
+
+        try:
+            response = self._transport.execute_sync(
+                SyncRequest(
+                    httpx_request.method,
+                    str(httpx_request.url),
+                    headers=request_headers,
+                    content=request_content,
+                    timeout=timeout,  # pyright: ignore[reportCallIssue]
+                )
+            )
+        except StreamError as e:
+            raise map_stream_error(e) from e
+
+        def get_trailers() -> httpx.Headers:
+            return httpx.Headers(tuple(response.trailers.items()))
+
+        return httpx.Response(
+            status_code=response.status,
+            headers=httpx.Headers(tuple(response.headers.items())),
+            stream=IteratorByteStream(response),
+            extensions={"get_trailers": get_trailers},
+        )
+
+
+def sync_request_content(
+    stream: httpx.AsyncByteStream | httpx.SyncByteStream | httpx.ByteStream,
+) -> bytes | Iterator[bytes]:
+    match stream:
+        case httpx.ByteStream():
+            # Buffered bytes
+            return next(iter(stream))
+        case _:
+            return sync_request_content_iter(stream)
+
+
+def sync_request_content_iter(
+    stream: httpx.AsyncByteStream | httpx.SyncByteStream,
+) -> Iterator[bytes]:
+    match stream:
+        case httpx.AsyncByteStream():
+            msg = "unreachable"
+            raise TypeError(msg)
+        case httpx.SyncByteStream():
+            with contextlib.closing(stream):
+                yield from stream
+
+
+class IteratorByteStream(httpx.SyncByteStream):
+    def __init__(self, response: SyncResponse) -> None:
+        self._response = response
+        self._is_stream_consumed = False
+
+    def __iter__(self) -> Iterator[bytes]:
+        if self._is_stream_consumed:
+            raise httpx.StreamConsumed
+        self._is_stream_consumed = True
+        try:
+            for chunk in self._response.content:
+                yield bytes(chunk)
+        except StreamError as e:
+            raise map_stream_error(e) from e
+
+    def close(self) -> None:
+        self._response.close()
+
+
+# Headers that are managed by the transport and should not be forwarded.
+TRANSPORT_HEADERS = {
+    "connection",
+    "keep-alive",
+    "proxy-connection",
+    "transfer-encoding",
+    "upgrade",
+}
+
+
+def convert_headers(headers: httpx.Headers) -> Headers:
+    return Headers(
+        (k, v) for k, v in headers.multi_items() if k.lower() not in TRANSPORT_HEADERS
+    )
+
+
+def convert_timeout(extensions: dict) -> float | None:
+    httpx_timeout = cast("dict | None", extensions.get("timeout"))
+    if httpx_timeout is None:
+        return None
+    # reqwest does not support setting individual timeout settings
+    # per call, only an operation timeout, so we need to approximate
+    # that from the httpx timeout dict. Connect usually happens once
+    # and can be given a longer timeout - we assume the operation timeout
+    # is the max of read/write if present, or connect if not. We ignore
+    # pool for now
+    read_timeout = httpx_timeout.get("read", -1)
+    if read_timeout is None:
+        read_timeout = -1
+    write_timeout = httpx_timeout.get("write", -1)
+    if write_timeout is None:
+        write_timeout = -1
+    operation_timeout = max(read_timeout, write_timeout)
+    if operation_timeout != -1:
+        return operation_timeout
+    return httpx_timeout.get("connect")
+
+
+def map_stream_error(e: StreamError) -> httpx.RemoteProtocolError:
+    match e.code:
+        case StreamErrorCode.NO_ERROR:
+            code = ErrorCodes.NO_ERROR
+        case StreamErrorCode.PROTOCOL_ERROR:
+            code = ErrorCodes.PROTOCOL_ERROR
+        case StreamErrorCode.INTERNAL_ERROR:
+            code = ErrorCodes.INTERNAL_ERROR
+        case StreamErrorCode.FLOW_CONTROL_ERROR:
+            code = ErrorCodes.FLOW_CONTROL_ERROR
+        case StreamErrorCode.SETTINGS_TIMEOUT:
+            code = ErrorCodes.SETTINGS_TIMEOUT
+        case StreamErrorCode.STREAM_CLOSED:
+            code = ErrorCodes.STREAM_CLOSED
+        case StreamErrorCode.FRAME_SIZE_ERROR:
+            code = ErrorCodes.FRAME_SIZE_ERROR
+        case StreamErrorCode.REFUSED_STREAM:
+            code = ErrorCodes.REFUSED_STREAM
+        case StreamErrorCode.CANCEL:
+            code = ErrorCodes.CANCEL
+        case StreamErrorCode.COMPRESSION_ERROR:
+            code = ErrorCodes.COMPRESSION_ERROR
+        case StreamErrorCode.CONNECT_ERROR:
+            code = ErrorCodes.CONNECT_ERROR
+        case StreamErrorCode.ENHANCE_YOUR_CALM:
+            code = ErrorCodes.ENHANCE_YOUR_CALM
+        case StreamErrorCode.INADEQUATE_SECURITY:
+            code = ErrorCodes.INADEQUATE_SECURITY
+        case StreamErrorCode.HTTP_1_1_REQUIRED:
+            code = ErrorCodes.HTTP_1_1_REQUIRED
+        case _:
+            code = ErrorCodes.INTERNAL_ERROR
+    return httpx.RemoteProtocolError(str(StreamReset(stream_id=-1, error_code=code)))

pyqwest/testing/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+__all__ = ["ASGITransport", "WSGITransport"]
+
+from ._asgi import ASGITransport
+from ._wsgi import WSGITransport

pyqwest/testing/_asgi.py

@@ -0,0 +1,366 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import math
+from collections.abc import AsyncIterator
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+from urllib.parse import unquote, urlparse
+
+from pyqwest import (
+    Headers,
+    HTTPVersion,
+    ReadError,
+    Request,
+    Response,
+    Transport,
+    WriteError,
+)
+
+from ._asgi_compatibility import guarantee_single_callable
+
+if TYPE_CHECKING:
+    from types import TracebackType
+
+    from asgiref.typing import (
+        ASGI3Application,
+        ASGIApplication,
+        ASGIReceiveEvent,
+        ASGISendEvent,
+        ASGIVersions,
+        HTTPScope,
+        LifespanScope,
+        LifespanShutdownEvent,
+        LifespanStartupEvent,
+    )
+
+_asgi: ASGIVersions = {"version": "3.0", "spec_version": "2.5"}
+_extensions = {"http.response.trailers": {}}
+
+
+@dataclass(frozen=True)
+class Lifespan:
+    task: asyncio.Task[None]
+    receive_queue: asyncio.Queue[LifespanStartupEvent | LifespanShutdownEvent]
+    send_queue: asyncio.Queue[ASGISendEvent | Exception]
+
+
+class ASGITransport(Transport):
+    """Transport implementation that directly invokes an ASGI application. Useful for testing.
+
+    The ASGI transport supports lifespan - to use it, make sure to use the transport as an
+    asynchronous context manager. Lifespan startup will be run on entering and shutdown when
+    exiting.
+    """
+
+    _app: ASGI3Application
+    _http_version: HTTPVersion
+    _client: tuple[str, int]
+    _state: dict[str, Any]
+    _lifespan: Lifespan | None
+
+    def __init__(
+        self,
+        app: ASGIApplication,
+        http_version: HTTPVersion = HTTPVersion.HTTP2,
+        client: tuple[str, int] = ("127.0.0.1", 111),
+    ) -> None:
+        """Creates a new ASGI transport.
+
+        Args:
+            app: The ASGI application to invoke.
+            http_version: The HTTP version to mimic for requests. Note, semantics such as lack of
+                          bidirectional streaming for HTTP/1 are not enforced.
+            client: The (host, port) tuple to use for the client address in the ASGI scope.
+        """
+        self._app = guarantee_single_callable(app)
+        self._http_version = http_version
+        self._client = client
+        self._state = {}
+        self._lifespan = None
+
+    async def execute(self, request: Request) -> Response:
+        parsed_url = urlparse(request.url)
+        raw_path = parsed_url.path or "/"
+        path = unquote(raw_path)
+        match self._http_version:
+            case HTTPVersion.HTTP1:
+                http_version = "1.1"
+            case HTTPVersion.HTTP2:
+                http_version = "2"
+            case HTTPVersion.HTTP3:
+                http_version = "3"
+            case _:
+                http_version = "1.1"
+        scope: HTTPScope = {
+            "type": "http",
+            "asgi": _asgi,
+            "http_version": http_version,
+            "method": request.method,
+            "scheme": parsed_url.scheme,
+            "path": path,
+            "raw_path": raw_path.encode(),
+            "query_string": parsed_url.query.encode(),
+            "headers": [
+                (k.lower().encode("utf-8"), v.encode("utf-8"))
+                for k, v in request.headers.items()
+            ],
+            "server": (
+                parsed_url.hostname or "",
+                parsed_url.port or (443 if parsed_url.scheme == "https" else 80),
+            ),
+            "client": self._client,
+            "extensions": _extensions,
+            "state": self._state,
+            "root_path": "",
+        }
+
+        receive_queue: asyncio.Queue[bytes | Exception | None] = asyncio.Queue(1)
+
+        async def read_request_content() -> None:
+            try:
+                async for chunk in request.content:
+                    if not isinstance(chunk, bytes):
+                        msg = "Request not bytes object"
+                        raise WriteError(msg)  # noqa: TRY301
+                    await receive_queue.put(chunk)
+                await receive_queue.put(None)
+            except Exception as e:
+                await receive_queue.put(e)
+            finally:
+                try:
+                    aclose = request.content.aclose  # pyright: ignore[reportAttributeAccessIssue]
+                except AttributeError:
+                    pass
+                else:
+                    await aclose()
+
+        # Need a separate task to read the request body to allow
+        # cancelling when response closes.
+        request_task = asyncio.create_task(read_request_content())
+
+        async def receive() -> ASGIReceiveEvent:
+            chunk = await receive_queue.get()
+            if chunk is None:
+                return {"type": "http.request", "body": b"", "more_body": False}
+            if isinstance(chunk, Exception):
+                if self._http_version != HTTPVersion.HTTP2:
+                    msg = f"Request failed: {chunk}"
+                else:
+                    # With HTTP/2, reqwest seems to squash the original error message.
+                    msg = "Request failed: stream error sent by user"
+                raise WriteError(msg) from chunk
+            if isinstance(chunk, BaseException):
+                raise chunk
+            return {"type": "http.request", "body": chunk, "more_body": True}
+
+        send_queue: asyncio.Queue[ASGISendEvent | Exception] = asyncio.Queue()
+
+        async def send(message: ASGISendEvent) -> None:
+            await send_queue.put(message)
+
+        timeout: float | None = request._timeout  # pyright: ignore[reportAttributeAccessIssue]  # noqa: SLF001
+        if timeout is not None and (timeout < 0 or not math.isfinite(timeout)):
+            msg = "Timeout must be non-negative"
+            raise ValueError(msg)
+
+        async def run_app() -> None:
+            try:
+                await asyncio.wait_for(self._app(scope, receive, send), timeout=timeout)
+            except asyncio.TimeoutError as e:
+                send_queue.put_nowait(TimeoutError(str(e)))
+            except Exception as e:
+                send_queue.put_nowait(e)
+
+        app_task = asyncio.create_task(run_app())
+        message = await send_queue.get()
+        if isinstance(message, Exception):
+            await app_task
+            if isinstance(message, TimeoutError):
+                raise message
+            return Response(
+                status=500,
+                http_version=self._http_version,
+                headers=Headers((("content-type", "text/plain"),)),
+                content=str(message).encode(),
+            )
+
+        assert message["type"] == "http.response.start"  # noqa: S101
+        status = message["status"]
+        headers = Headers(
+            (
+                (k.decode("utf-8"), v.decode("utf-8"))
+                for k, v in message.get("headers", [])
+            )
+        )
+        trailers = (
+            Headers()
+            if self._http_version == HTTPVersion.HTTP2
+            and request.headers.get("te") == "trailers"
+            else None
+        )
+        response_content = ResponseContent(
+            send_queue,
+            request_task,
+            trailers,
+            app_task,
+            read_trailers=message.get("trailers", False),
+        )
+        return Response(
+            status=status,
+            http_version=self._http_version,
+            headers=headers,
+            content=response_content,
+            trailers=trailers,
+        )
+
+    async def __aenter__(self) -> ASGITransport:
+        await self.run_lifespan()
+        return self
+
+    async def __aexit__(
+        self,
+        _exc_type: type[BaseException] | None,
+        _exc_value: BaseException | None,
+        _traceback: TracebackType | None,
+    ) -> None:
+        await self.close()
+
+    async def run_lifespan(self) -> None:
+        scope: LifespanScope = {"type": "lifespan", "asgi": _asgi, "state": self._state}
+
+        receive_queue: asyncio.Queue[LifespanStartupEvent | LifespanShutdownEvent] = (
+            asyncio.Queue()
+        )
+
+        async def receive() -> LifespanStartupEvent | LifespanShutdownEvent:
+            return await receive_queue.get()
+
+        send_queue: asyncio.Queue[ASGISendEvent | Exception] = asyncio.Queue()
+
+        async def send(message: ASGISendEvent) -> None:
+            await send_queue.put(message)
+
+        async def run_app() -> None:
+            try:
+                await self._app(scope, receive, send)
+            except Exception as e:
+                send_queue.put_nowait(e)
+
+        task = asyncio.create_task(run_app())
+
+        receive_queue.put_nowait({"type": "lifespan.startup"})
+        message = await send_queue.get()
+        if isinstance(message, Exception):
+            # Lifespan not supported
+            await task
+            return
+
+        self._lifespan = Lifespan(
+            task=task, receive_queue=receive_queue, send_queue=send_queue
+        )
+        match message["type"]:
+            case "lifespan.startup.complete":
+                return
+            case "lifespan.startup.failed":
+                msg = "ASGI application failed to start up"
+                raise RuntimeError(msg)
+
+    async def close(self) -> None:
+        if self._lifespan is None:
+            return
+
+        await self._lifespan.receive_queue.put({"type": "lifespan.shutdown"})
+        message = await self._lifespan.send_queue.get()
+        await self._lifespan.task
+        if isinstance(message, Exception):
+            raise message
+        match message["type"]:
+            case "lifespan.shutdown.complete":
+                return
+            case "lifespan.shutdown.failed":
+                msg = "ASGI application failed to shut down cleanly"
+                raise RuntimeError(msg)
+
+
+class CancelResponse(Exception):
+    pass
+
+
+class ResponseContent(AsyncIterator[bytes]):
+    def __init__(
+        self,
+        send_queue: asyncio.Queue[ASGISendEvent | Exception],
+        request_task: asyncio.Task[None],
+        trailers: Headers | None,
+        task: asyncio.Task[None],
+        *,
+        read_trailers: bool,
+    ) -> None:
+        self._send_queue = send_queue
+        self._request_task = request_task
+        self._trailers = trailers
+        self._task = task
+        self._read_trailers = read_trailers
+
+        self._read_pending = False
+        self._closed = False
+
+    def __aiter__(self) -> AsyncIterator[bytes]:
+        return self
+
+    async def __anext__(self) -> bytes:
+        if self._closed:
+            raise StopAsyncIteration
+        err: Exception | None = None
+        body: bytes | None = None
+        while True:
+            self._read_pending = True
+            try:
+                message = await self._send_queue.get()
+            finally:
+                self._read_pending = False
+            if isinstance(message, Exception):
+                match message:
+                    case CancelResponse():
+                        err = StopAsyncIteration()
+                        break
+                    case WriteError() | TimeoutError():
+                        err = message
+                        break
+                    case ReadError():
+                        raise message
+                    case Exception():
+                        msg = "Error reading response body"
+                        raise ReadError(msg) from message
+            match message["type"]:
+                case "http.response.body":
+                    if not message.get("more_body", False) and not self._read_trailers:
+                        await self._cleanup()
+                    if (body := message.get("body", b"")) or self._closed:
+                        return body
+                case "http.response.trailers":
+                    if self._trailers is not None:
+                        for k, v in message.get("headers", []):
+                            self._trailers.add(k.decode("utf-8"), v.decode("utf-8"))
+                    if not message.get("more_trailers", False):
+                        break
+        await self._cleanup()
+        if err:
+            raise err
+        raise StopAsyncIteration
+
+    async def aclose(self) -> None:
+        if self._closed:
+            return
+        self._closed = True
+        self._send_queue.put_nowait(ReadError("Response body read cancelled"))
+        await self._cleanup()
+
+    async def _cleanup(self) -> None:
+        self._closed = True
+        self._request_task.cancel()
+        with contextlib.suppress(BaseException):
+            await self._request_task
+            await self._task