wpr-http 0.2.0__py3-none-any.whl

wpr_http/exceptions.py

@@ -0,0 +1,207 @@
+"""HTTP client exception hierarchy and Gecko network error translation."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from ._urls import safe_proxy, safe_url
+
+_GECKO_ERRORS: dict[int, tuple[type[WprHttpError], str]] = {}
+
+
+class WprHttpError(RuntimeError):
+    """Base exception, analogous to ``httpx.HTTPError``."""
+
+    def __init__(
+        self,
+        response: Mapping[str, Any],
+        *,
+        method: str | None = None,
+        url: str | None = None,
+        proxy: str | None = None,
+        request: Any = None,
+        message: str | None = None,
+        status_code: int | None = None,
+    ) -> None:
+        error = response.get("error")
+        self.worker_response = dict(response)
+        self.response = None
+        self.request = request
+        # True only when the client-side deadline/cancellation stopped the
+        # Worker process. Gecko-reported network timeouts leave it reusable.
+        self.worker_stopped = False
+        self.code = (
+            str(error.get("code", "HTTP_ERROR"))
+            if isinstance(error, Mapping)
+            else str(error or "HTTP_ERROR")
+        )
+        self.rv = error.get("rv") if isinstance(error, Mapping) else None
+        try:
+            self.rv_hex = f"0x{int(self.rv):08X}" if self.rv is not None else None
+        except (TypeError, ValueError):
+            self.rv_hex = None
+        self.method = method.upper() if method else None
+        self.url = safe_url(url) if url else None
+        self.proxy = safe_proxy(proxy) if proxy else None
+        self.status_code = status_code
+        if message is None:
+            request = (
+                f"{self.method} {self.url}" if self.method and self.url else "request"
+            )
+            message = f"{request} failed: {_rv_description(self.rv)}"
+            if self.proxy:
+                message += f" (proxy {self.proxy})"
+            message += "; no HTTP response was received"
+            message += f" [{self.code}"
+            if self.rv_hex:
+                message += f", rv={self.rv_hex}"
+            message += "]"
+        super().__init__(message)
+
+
+class WprTransportError(WprHttpError):
+    """Base for failures before response headers arrive."""
+
+
+class WprConnectError(WprTransportError):
+    """Connection, DNS, or socket failure."""
+
+
+class WprProxyError(WprConnectError):
+    """Proxy connection, name-resolution, or authentication failure."""
+
+
+class WprTimeoutError(WprTransportError):
+    """Request exceeded its configured deadline."""
+
+
+class WprPoolTimeoutError(WprTimeoutError):
+    """Request exceeded the deadline while waiting for a Worker pool slot."""
+
+
+class WprHTTPStatusError(WprHttpError):
+    """HTTP 4xx/5xx response raised by ``HttpResponse.raise_for_status``."""
+
+    def __init__(self, response: Any) -> None:
+        self.response = response
+        self.request = response.request
+        self.worker_response = None
+        self.status_code = response.status_code
+        self.code = "HTTP_STATUS_ERROR"
+        self.rv = self.rv_hex = self.proxy = None
+        self.method = response.request_method
+        self.url = safe_url(response.url)
+        request = f" for {self.method} {self.url}" if self.method and self.url else ""
+        reason = f" {response.reason}" if response.reason else ""
+        RuntimeError.__init__(self, f"HTTP {self.status_code}{reason}{request}")
+
+
+_GECKO_ERRORS.update(
+    {
+        0x804B000D: (
+            WprConnectError,
+            "connection refused by the peer (destination or proxy upstream)",
+        ),
+        0x804B0014: (WprConnectError, "connection was reset before an HTTP response"),
+        0x804B001E: (WprConnectError, "DNS lookup failed for the destination host"),
+        0x804B002A: (WprProxyError, "DNS lookup failed for the proxy host"),
+        0x804B0048: (WprProxyError, "connection to the proxy server was refused"),
+        0x804B057F: (
+            WprProxyError,
+            "the proxy requires authentication or rejected its credentials",
+        ),
+        0x804B000E: (WprTimeoutError, "network connection timed out"),
+        0x804B0055: (WprTimeoutError, "network request timed out"),
+        0x804B05E0: (WprTimeoutError, "the proxy gateway timed out"),
+    }
+)
+
+
+def _rv_description(rv: Any) -> str:
+    try:
+        mapped = _GECKO_ERRORS.get(int(rv))
+    except (TypeError, ValueError):
+        mapped = None
+    return (
+        mapped[1]
+        if mapped
+        else "network request failed before receiving an HTTP response"
+    )
+
+
+def transport_error(
+    response: Mapping[str, Any],
+    *,
+    method: str,
+    url: str,
+    proxy: str | None,
+    request: Any = None,
+) -> WprHttpError:
+    error = response.get("error")
+    rv = error.get("rv") if isinstance(error, Mapping) else None
+    try:
+        error_type = _GECKO_ERRORS.get(int(rv), (WprTransportError, ""))[0]
+    except (TypeError, ValueError):
+        error_type = WprTransportError
+    return error_type(response, method=method, url=url, proxy=proxy, request=request)
+
+
+def timeout_error(
+    method: str,
+    url: str,
+    timeout: float,
+    *,
+    request: Any = None,
+    worker_stopped: bool = True,
+) -> WprTimeoutError:
+    error = WprTimeoutError(
+        {"error": {"code": "REQUEST_TIMEOUT"}},
+        method=method,
+        url=url,
+        request=request,
+        message=(
+            f"{method.upper()} {safe_url(url)} exceeded the total timeout "
+            f"of {timeout:.3g}s; "
+            + (
+                "the network worker was stopped and will restart for the next request"
+                if worker_stopped
+                else "the request channel was cancelled; other requests remain active"
+            )
+            + " [REQUEST_TIMEOUT]"
+        ),
+    )
+    error.worker_stopped = worker_stopped
+    return error
+
+
+def pool_timeout_error(
+    method: str, url: str, timeout: float, *, request: Any = None
+) -> WprPoolTimeoutError:
+    return WprPoolTimeoutError(
+        {"error": {"code": "POOL_TIMEOUT"}},
+        method=method,
+        url=url,
+        request=request,
+        message=(
+            f"{method.upper()} {safe_url(url)} timed out after {timeout:.3g}s "
+            "waiting for an available Worker slot; active requests "
+            "were left running [POOL_TIMEOUT]"
+        ),
+    )
+
+
+def dispatch_timeout_error(
+    method: str, url: str, timeout: float, *, request: Any = None
+) -> WprTimeoutError:
+    return WprTimeoutError(
+        {"error": {"code": "REQUEST_TIMEOUT"}},
+        method=method,
+        url=url,
+        request=request,
+        message=(
+            f"{method.upper()} {safe_url(url)} exceeded its total timeout of "
+            f"{timeout:.3g}s while waiting to dispatch; the active Worker was left running "
+            "[REQUEST_TIMEOUT]"
+        ),
+    )

wpr_http/models.py

@@ -0,0 +1,331 @@
+"""Request timeout and response models for the network client."""
+
+from __future__ import annotations
+
+import codecs
+import json
+import logging
+import math
+from collections.abc import Iterable, Mapping, MutableMapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from wpr_runtime_core.data_plane import SharedBlob, SharedBlobDescriptor
+from wpr_runtime_core.protocols import BodyLease, ReleaseSession
+
+from ._urls import safe_url
+from .exceptions import WprHTTPStatusError
+
+_logger = logging.getLogger("wpr_http")
+
+
+@dataclass(frozen=True)
+class Timeout:
+    """End-to-end request timeout in seconds.
+
+    ``total`` covers queueing plus the active request. ``pool`` separately
+    limits waiting for an available Worker slot. ``None``
+    disables the corresponding deadline. Socket phase limits require Worker
+    protocol support.
+    """
+
+    total: float | None = 30.0
+    pool: float | None = None
+
+    def __post_init__(self) -> None:
+        for name in ("total", "pool"):
+            value = getattr(self, name)
+            if value is not None and (not math.isfinite(value) or value < 0):
+                raise ValueError(f"timeout.{name} must be finite and >= 0, or None")
+
+
+@dataclass(frozen=True)
+class NetworkTiming:
+    request_start_ms: int
+    response_headers_ms: int
+    response_end_ms: int
+
+
+class Headers(MutableMapping[str, str]):
+    """Case-insensitive header mapping that also preserves repeated values."""
+
+    def __init__(
+        self,
+        values: Mapping[str, str] | Iterable[tuple[str, str]] | None = None,
+    ) -> None:
+        self._values: dict[str, tuple[str, list[str]]] = {}
+        source = values.items() if isinstance(values, Mapping) else (values or ())
+        for name, value in source:
+            self.add(str(name), str(value))
+
+    def add(self, name: str, value: str) -> None:
+        key = name.lower()
+        if key in self._values:
+            original, values = self._values[key]
+            values.append(str(value))
+            self._values[key] = (original, values)
+        else:
+            self._values[key] = (str(name), [str(value)])
+
+    def get_list(self, name: str) -> list[str]:
+        value = self._values.get(name.lower())
+        return list(value[1]) if value else []
+
+    def __getitem__(self, name: str) -> str:
+        try:
+            values = self._values[name.lower()][1]
+        except KeyError:
+            raise KeyError(name) from None
+        return ", ".join(values)
+
+    def __setitem__(self, name: str, value: str) -> None:
+        self._values[name.lower()] = (str(name), [str(value)])
+
+    def __delitem__(self, name: str) -> None:
+        del self._values[name.lower()]
+
+    def __iter__(self):
+        return (original for original, _values in self._values.values())
+
+    def __len__(self) -> int:
+        return len(self._values)
+
+
+@dataclass(frozen=True)
+class Request:
+    """Fully prepared request, separate from sending it through a Worker."""
+
+    method: str
+    url: str
+    headers: Mapping[str, str] = field(default_factory=Headers)
+    content: str | None = None
+    timeout: Timeout | float | None = field(default_factory=Timeout)
+    follow_redirects: bool = True
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "method", self.method.upper())
+        object.__setattr__(self, "headers", Headers(self.headers))
+        if not isinstance(self.timeout, Timeout):
+            value = None if self.timeout is None else float(self.timeout)
+            object.__setattr__(self, "timeout", Timeout(total=value))
+
+    def __repr__(self) -> str:
+        return f"<Request [{self.method}] {safe_url(self.url)}>"
+
+
+HttpRequest = Request
+
+
+class HttpResponse:
+    """Response metadata and a shared-memory body lease."""
+
+    def __init__(
+        self,
+        session: ReleaseSession | None,
+        value: Mapping[str, Any],
+        *,
+        request: Request | None = None,
+        request_method: str | None = None,
+        request_url: str | None = None,
+        rpc_timings: Mapping[str, Any] | None = None,
+        body: BodyLease | None = None,
+    ) -> None:
+        self._session = session
+        self.request = request or Request(
+            method=request_method or "GET",
+            url=request_url or str(value.get("final_url", "")),
+            headers={},
+            content=None,
+            timeout=Timeout(total=None),
+            follow_redirects=True,
+        )
+        self.request_method = self.request.method
+        self.request_url = self.request.url
+        self.status_code = int(value.get("status", 0))
+        self.reason = str(value.get("status_text", ""))
+        self.url = str(value.get("final_url", ""))
+        self.headers = _headers(value.get("headers"))
+        self.raw_headers = tuple(
+            (str(item.get("name", "")), str(item.get("value", "")))
+            for item in value.get("headers", [])
+            if isinstance(item, Mapping)
+        )
+        self.redirects = tuple(value.get("redirects") or ())
+        timing = value.get("timings") or {}
+        self.timing = NetworkTiming(
+            int(timing.get("request_start_ms", 0)),
+            int(timing.get("response_headers_ms", 0)),
+            int(timing.get("response_end_ms", 0)),
+        )
+        if body is None:
+            if session is None:
+                raise ValueError("response requires a body lease or a release session")
+            body = SharedBlob(SharedBlobDescriptor.from_value(value["body"]), session)
+        self._body = body
+        # Attach to the named mapping before the client releases this Worker
+        # slot. If the Worker later exits, Windows retains the section through
+        # this local mapping until the response closes.
+        self._body.open()
+        self._content: bytes | None = None
+        self._on_close = None
+        self._closed = False
+        self.encoding = _response_encoding(self.headers, self._body.descriptor.encoding)
+        self._wpr_diagnostics: dict[str, Any] = {
+            "fetch_rpc": dict(rpc_timings or {}),
+            "worker": dict(value.get("_wpr_timing") or {}),
+            "mmap_open_ms": self._body.mapping_open_ms,
+        }
+
+    @property
+    def content(self) -> bytes:
+        if self._content is None:
+            if self._closed:
+                raise RuntimeError("response body is closed")
+            self._body.open()
+            self._content = self._body.read_bytes()
+            self._wpr_diagnostics["body_copy_ms"] = self._body.copy_ms
+        return self._content
+
+    @property
+    def is_success(self) -> bool:
+        return 200 <= self.status_code < 300
+
+    @property
+    def ok(self) -> bool:
+        return self.status_code < 400
+
+    @property
+    def is_redirect(self) -> bool:
+        return 300 <= self.status_code < 400
+
+    @property
+    def is_error(self) -> bool:
+        return self.status_code >= 400
+
+    @property
+    def is_closed(self) -> bool:
+        return self._closed
+
+    async def aread(self) -> bytes:
+        data = self.content
+        await self.aclose()
+        return data
+
+    async def read(self) -> bytes:
+        """Read and release the shared response body."""
+        return await self.aread()
+
+    async def atext(self, encoding: str | None = None) -> str:
+        return (await self.aread()).decode(encoding or self.encoding)
+
+    async def text(self, encoding: str | None = None) -> str:
+        return await self.atext(encoding)
+
+    async def ajson(self) -> Any:
+        return json.loads(await self.atext())
+
+    async def json(self) -> Any:
+        return await self.ajson()
+
+    def raise_for_status(self) -> None:
+        if self.status_code >= 400:
+            _logger.warning(
+                "http.status_error status_code=%s method=%s url=%s",
+                self.status_code,
+                self.request_method or "?",
+                safe_url(self.url),
+            )
+            raise WprHTTPStatusError(self)
+
+    async def aclose(self) -> None:
+        if self._closed:
+            return
+        try:
+            await self._body.release()
+            self._wpr_diagnostics["mmap_open_ms"] = self._body.mapping_open_ms
+            self._wpr_diagnostics["body_copy_ms"] = self._body.copy_ms
+            self._wpr_diagnostics["release_rpc"] = dict(self._body.release_timings)
+        finally:
+            self._closed = True
+            if self._on_close is not None:
+                self._on_close(self)
+
+    def _set_on_close(self, callback) -> None:
+        self._on_close = callback
+
+    async def __aenter__(self) -> "HttpResponse":  # noqa: PYI034, UP037
+        self._body.open()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        await self.aclose()
+
+    def __repr__(self) -> str:
+        reason = f" {self.reason}" if self.reason else ""
+        return f"<HttpResponse [{self.status_code}{reason}]>"
+
+    @property
+    def diagnostics(self) -> dict[str, Any]:
+        """Return measured client/RPC phases and optional Worker phases."""
+        return {
+            **self._wpr_diagnostics,
+            "fetch_rpc": dict(self._wpr_diagnostics.get("fetch_rpc") or {}),
+            "worker": dict(self._wpr_diagnostics.get("worker") or {}),
+            "release_rpc": dict(self._wpr_diagnostics.get("release_rpc") or {}),
+        }
+
+
+def _headers(value: Any) -> Headers:
+    result = Headers()
+    if isinstance(value, list):
+        for item in value:
+            if isinstance(item, Mapping):
+                name = str(item.get("name", "")).lower()
+                if name:
+                    result.add(name, str(item.get("value", "")))
+    return result
+
+
+def _response_encoding(headers: Headers, fallback: str) -> str:
+    from email.message import Message
+
+    for content_type in headers.get_list("content-type"):
+        for value in _split_quoted_values(content_type, ","):
+            message = Message()
+            message["content-type"] = value
+            charset = message.get_content_charset()
+            if charset:
+                encoding = _known_encoding(charset)
+                if encoding:
+                    return encoding
+
+    return _known_encoding(fallback) or "utf-8"
+
+
+def _split_quoted_values(value: str, separator: str) -> list[str]:
+    """Split a combined header value without breaking quoted parameters."""
+    parts: list[str] = []
+    start = 0
+    quoted = False
+    escaped = False
+    for index, character in enumerate(value):
+        if escaped:
+            escaped = False
+        elif quoted and character == "\\":
+            escaped = True
+        elif character == '"':
+            quoted = not quoted
+        elif character == separator and not quoted:
+            parts.append(value[start:index].strip())
+            start = index + 1
+    parts.append(value[start:].strip())
+    return [part for part in parts if part]
+
+
+def _known_encoding(value: str | None) -> str | None:
+    if not value:
+        return None
+    try:
+        return codecs.lookup(value.strip()).name
+    except LookupError:
+        return None

wpr_http/transport.py

@@ -0,0 +1,121 @@
+"""Adapter between the stable HTTP API and the versioned Gecko Worker."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from wpr_runtime_core.data_plane import SharedBlob, SharedBlobDescriptor
+from wpr_runtime_core.session import AsyncWorkerSession as AsyncGeckoWorkerSession
+
+from ._urls import safe_url
+from .exceptions import timeout_error, transport_error
+from .models import HttpResponse, Request, Timeout
+
+_logger = logging.getLogger("wpr_http")
+
+
+class GeckoNetworkTransport:
+    """One-request adapter; all Gecko protocol details stay below the client."""
+
+    async def send(
+        self,
+        session: AsyncGeckoWorkerSession,
+        *,
+        request: Request,
+        proxy: str | None,
+        timeout: Timeout | None = None,
+    ) -> HttpResponse:
+        active_timeout = timeout or request.timeout
+        fetch_kwargs = {
+            "method": request.method,
+            "headers": dict(request.headers),
+            "body": request.content,
+            "follow_redirects": request.follow_redirects,
+        }
+        timed_fetch = getattr(session, "network_fetch_with_timing", None)
+        if timed_fetch is None:
+
+            async def legacy_fetch():
+                return await session.network_fetch(request.url, **fetch_kwargs), {}
+
+            operation = legacy_fetch()
+        else:
+            operation = timed_fetch(request.url, **fetch_kwargs)
+        try:
+            if active_timeout.total is None:
+                raw, rpc_timings = await operation
+            else:
+                raw, rpc_timings = await asyncio.wait_for(
+                    operation, active_timeout.total
+                )
+        except asyncio.TimeoutError as exc:
+            # Multiplex cancellation completes in the IOCP request coroutine;
+            # legacy synchronous requests still require stopping the Worker.
+            granular = (
+                getattr(session, "supports_request_cancel", False) and session.is_open
+            )
+            if not granular:
+                await session.abort()
+            timeout_exc = timeout_error(
+                request.method,
+                request.url,
+                active_timeout.total or 0,
+                request=request,
+                worker_stopped=not granular,
+            )
+            _logger.warning(
+                "http.timeout method=%s url=%s timeout_total=%s status_code=None "
+                "worker_action=%s message=%s",
+                request.method,
+                timeout_exc.url,
+                round(active_timeout.total, 3)
+                if active_timeout.total is not None
+                else None,
+                "cancel_channel" if granular else "abort_and_restart",
+                str(timeout_exc),
+            )
+            raise timeout_exc from exc
+        except asyncio.CancelledError:
+            # Keep multiplex siblings alive, but unblock legacy executor reads
+            # by stopping their Worker.
+            granular = (
+                getattr(session, "supports_request_cancel", False) and session.is_open
+            )
+            if not granular:
+                await session.abort()
+            _logger.warning(
+                "http.cancelled method=%s url=%s worker_action=%s",
+                request.method,
+                safe_url(request.url),
+                "cancel_channel" if granular else "abort_and_restart",
+            )
+            raise
+
+        if not raw.get("ok"):
+            exc = transport_error(
+                raw,
+                method=request.method,
+                url=request.url,
+                proxy=proxy,
+                request=request,
+            )
+            _logger.warning(
+                "http.transport_error method=%s url=%s status_code=None "
+                "exception=%s code=%s rv=%s proxy=%s message=%s",
+                request.method,
+                exc.url,
+                type(exc).__name__,
+                exc.code,
+                exc.rv_hex or exc.rv,
+                exc.proxy or "none",
+                str(exc),
+            )
+            raise exc
+        return HttpResponse(
+            session,
+            raw,
+            request=request,
+            rpc_timings=rpc_timings,
+            body=SharedBlob(SharedBlobDescriptor.from_value(raw["body"]), session),
+        )