unique-search-proxy-core 2026.44.0.dev6__tar.gz → 2026.44.0.dev8__tar.gz

{unique_search_proxy_core-2026.44.0.dev6 → unique_search_proxy_core-2026.44.0.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy-core
-Version: 2026.44.0.dev6
+Version: 2026.44.0.dev8
 Summary: Shared Pydantic types for the Unique Search Proxy API
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>

{unique_search_proxy_core-2026.44.0.dev6 → unique_search_proxy_core-2026.44.0.dev8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy-core"
-version = "2026.44.0.dev6"
+version = "2026.44.0.dev8"
 description = "Shared Pydantic types for the Unique Search Proxy API"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"

{unique_search_proxy_core-2026.44.0.dev6 → unique_search_proxy_core-2026.44.0.dev8}/unique_search_proxy_core/errors.py

@@ -132,3 +132,15 @@ class EmptySearchResultsError(ProxyError):
     code = ProxyErrorCode.EMPTY_SEARCH_RESULTS
     status_code = 404
     retryable = False
+
+
+class ResponseTooLargeError(ProxyError):
+    """An upstream response body exceeded the configured byte limit."""
+
+    code = ProxyErrorCode.RESPONSE_TOO_LARGE
+    status_code = 502
+    retryable = False
+
+    @classmethod
+    def for_limit(cls, max_bytes: int) -> ResponseTooLargeError:
+        return cls(f"Upstream response exceeded the {max_bytes} byte limit")

{unique_search_proxy_core-2026.44.0.dev6 → unique_search_proxy_core-2026.44.0.dev8}/unique_search_proxy_core/http_client/__init__.py

@@ -17,6 +17,12 @@ from unique_search_proxy_core.http_client.credentials import (
     UserMetadataProxyCredentials,
     resolver_from_settings,
 )
+from unique_search_proxy_core.http_client.response_cap import (
+    MAX_RESPONSE_BYTES_EXTENSION,
+    install_response_size_cap,
+    read_capped_body,
+    response_size_cap_hook,
+)
 from unique_search_proxy_core.http_client.secrets import (
     read_secret,
     read_secret_mapping,
@@ -29,6 +35,7 @@ from unique_search_proxy_core.http_client.settings import (
 )
 
 __all__ = [
+    "MAX_RESPONSE_BYTES_EXTENSION",
     "DirectRoute",
     "EgressRoute",
     "HttpClientRegistry",
@@ -45,7 +52,10 @@ __all__ = [
     "async_client_factory",
     "build_async_client",
     "build_route",
+    "install_response_size_cap",
+    "read_capped_body",
     "read_secret",
     "read_secret_mapping",
     "resolver_from_settings",
+    "response_size_cap_hook",
 ]

{unique_search_proxy_core-2026.44.0.dev6 → unique_search_proxy_core-2026.44.0.dev8}/unique_search_proxy_core/http_client/client.py

@@ -20,6 +20,10 @@ from unique_search_proxy_core.http_client.credentials import (
     SettingsProxyCredentials,
     resolver_from_settings,
 )
+from unique_search_proxy_core.http_client.response_cap import (
+    ResponseHook,
+    response_size_cap_hook,
+)
 from unique_search_proxy_core.http_client.secrets import read_secret_mapping
 from unique_search_proxy_core.http_client.settings import (
     ProxyAuthMode,
@@ -96,6 +100,10 @@ def _proxy_headers(settings: ProxySettings) -> dict[str, str]:
     return read_secret_mapping(settings.proxy_headers)
 
 
+def _event_hooks(settings: ProxySettings) -> dict[str, list[ResponseHook]]:
+    return {"response": [response_size_cap_hook(settings.max_response_bytes)]}
+
+
 def _build_httpx_proxy(
     settings: ProxySettings,
     credentials: ProxyCredentials,
@@ -174,6 +182,7 @@ def build_async_client(
                 trust_env=route.trust_env,
                 timeout=timeout,
                 limits=limits,
+                event_hooks=_event_hooks(settings),
             )
         case ProxiedRoute():
             return AsyncClient(
@@ -184,6 +193,7 @@ def build_async_client(
                 cert=route.cert,
                 timeout=timeout,
                 limits=limits,
+                event_hooks=_event_hooks(settings),
             )
 
 
@@ -207,6 +217,7 @@ def async_client_factory(
                 verify=route.verify,
                 trust_env=route.trust_env,
                 timeout=effective_timeout,
+                event_hooks=_event_hooks(settings),
             )
         case ProxiedRoute():
             return partial(
@@ -217,6 +228,7 @@ def async_client_factory(
                 trust_env=route.trust_env,
                 cert=route.cert,
                 timeout=effective_timeout,
+                event_hooks=_event_hooks(settings),
             )
 
 
@@ -283,6 +295,10 @@ class HttpClientRegistry:
     ) -> None:
         if fixed_client is None:
             _validate_url_safety_proxy_configuration(settings)
+            _LOGGER.info(
+                "Outbound response size limit: %s bytes",
+                settings.max_response_bytes or "unlimited",
+            )
         self._settings = settings
         self._resolver = resolver
         self._fixed_client = fixed_client

unique_search_proxy_core-2026.44.0.dev8/unique_search_proxy_core/http_client/response_cap.py

@@ -0,0 +1,155 @@
+"""Bound how many bytes an outbound response may deliver.
+
+The limit is enforced while the body streams in, so an oversized, chunked, or
+endless response is aborted before it is buffered in memory. It is installed as
+an httpx response hook rather than a custom transport: a custom transport makes
+httpx ignore ``trust_env`` proxy discovery and every transport-level option.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Awaitable, Callable
+
+import httpx
+
+from unique_search_proxy_core.errors import ResponseTooLargeError
+
+MAX_RESPONSE_BYTES_EXTENSION = "max_response_bytes"
+
+# Raw chunks are re-sliced before httpx decompresses them, which bounds how much
+# a single gzip/deflate decode step can expand.
+_RAW_SLICE_BYTES = 16 * 1024
+
+ResponseHook = Callable[[httpx.Response], Awaitable[None]]
+
+
+class _CappedRawStream(httpx.AsyncByteStream):
+    """Wrap a response byte stream and abort once it delivers too many bytes.
+
+    Counts bytes as received from the network (before decompression), so it
+    also catches responses with a missing or understated ``Content-Length``.
+    """
+
+    def __init__(self, stream: httpx.AsyncByteStream, *, max_bytes: int) -> None:
+        self._stream = stream
+        self._max_bytes = max_bytes
+
+    async def __aiter__(self) -> AsyncIterator[bytes]:
+        received = 0
+        async for chunk in self._stream:
+            received += len(chunk)
+            if received > self._max_bytes:
+                raise ResponseTooLargeError.for_limit(self._max_bytes)
+            for start in range(0, len(chunk), _RAW_SLICE_BYTES):
+                yield chunk[start : start + _RAW_SLICE_BYTES]
+
+    async def aclose(self) -> None:
+        await self._stream.aclose()
+
+
+def _effective_max_bytes(
+    request: httpx.Request,
+    default_max_bytes: int | None,
+) -> int | None:
+    """Combine the client default with a per-request override; lower wins."""
+    override = request.extensions.get(MAX_RESPONSE_BYTES_EXTENSION)
+    if override is None:
+        return default_max_bytes
+    if default_max_bytes is None:
+        return int(override)
+    return min(int(override), default_max_bytes)
+
+
+def _declared_content_length(response: httpx.Response) -> int | None:
+    header = response.headers.get("content-length")
+    if header is None:
+        return None
+    try:
+        return int(header)
+    except ValueError:
+        return None
+
+
+def response_size_cap_hook(default_max_bytes: int | None) -> ResponseHook:
+    """Build an httpx response hook that caps the response body size.
+
+    Requests can lower the limit with
+    ``extensions={"max_response_bytes": n}``; they cannot raise it above
+    ``default_max_bytes``.
+
+    Args:
+        default_max_bytes (int | None): Limit applied when a request sets no
+            override. ``None`` leaves such requests uncapped.
+
+    Returns:
+        ResponseHook: Coroutine to register under ``event_hooks["response"]``.
+    """
+
+    async def _cap_response_size(response: httpx.Response) -> None:
+        max_bytes = _effective_max_bytes(response.request, default_max_bytes)
+        if max_bytes is None:
+            return
+        declared = _declared_content_length(response)
+        if declared is not None and declared > max_bytes:
+            raise ResponseTooLargeError.for_limit(max_bytes)
+        if isinstance(response.stream, httpx.AsyncByteStream):
+            response.stream = _CappedRawStream(response.stream, max_bytes=max_bytes)
+
+    return _cap_response_size
+
+
+def install_response_size_cap(
+    client: httpx.AsyncClient,
+    max_bytes: int | None,
+) -> httpx.AsyncClient:
+    """Register the response size cap on an existing client.
+
+    Args:
+        client (httpx.AsyncClient): Client to protect, e.g. a test client.
+        max_bytes (int | None): Default limit for the client's requests.
+
+    Returns:
+        httpx.AsyncClient: The same client, for chaining.
+    """
+    client.event_hooks["response"].append(response_size_cap_hook(max_bytes))
+    return client
+
+
+async def read_capped_body(
+    response: httpx.Response,
+    *,
+    max_bytes: int | None,
+) -> bytes:
+    """Read a streamed response body, limiting its decompressed size.
+
+    The response hook limits bytes received from the network; this limits the
+    bytes after gzip/deflate decoding, which stops decompression bombs.
+
+    Args:
+        response (httpx.Response): Response opened with ``stream=True``.
+        max_bytes (int | None): Largest decoded body allowed. ``None`` reads
+            the whole body.
+
+    Returns:
+        bytes: The decoded response body.
+
+    Raises:
+        ResponseTooLargeError: When the decoded body exceeds ``max_bytes``.
+    """
+    chunks: list[bytes] = []
+    received = 0
+    async for chunk in response.aiter_bytes():
+        received += len(chunk)
+        if max_bytes is not None and received > max_bytes:
+            raise ResponseTooLargeError.for_limit(max_bytes)
+        chunks.append(chunk)
+    return b"".join(chunks)
+
+
+__all__ = [
+    "MAX_RESPONSE_BYTES_EXTENSION",
+    "ResponseHook",
+    "install_response_size_cap",
+    "read_capped_body",
+    "response_size_cap_hook",
+]

{unique_search_proxy_core-2026.44.0.dev6 → unique_search_proxy_core-2026.44.0.dev8}/unique_search_proxy_core/http_client/settings.py

@@ -59,6 +59,22 @@ class ProxySettings(BaseSettings):
     pool_timeout_seconds: float = 30.0
     max_connections: int = 100
     max_keepalive_connections: int = 20
+    # KiB rather than bytes: Helm renders numbers >= 1e6 in scientific notation,
+    # which pydantic cannot parse back into an int.
+    max_response_kib: int | None = Field(
+        default=5 * 1024,
+        ge=1,
+        description=(
+            "Largest upstream response body, in KiB, any outbound request may "
+            "deliver before it is aborted. None disables the limit."
+        ),
+    )
+
+    @property
+    def max_response_bytes(self) -> int | None:
+        if self.max_response_kib is None:
+            return None
+        return self.max_response_kib * 1024
 
     @model_validator(mode="after")
     def _validate_proxy_credentials_config(self) -> Self:

{unique_search_proxy_core-2026.44.0.dev6 → unique_search_proxy_core-2026.44.0.dev8}/unique_search_proxy_core/schema.py

@@ -52,6 +52,7 @@ class ProxyErrorCode(StrEnum):
     ENGINE_NOT_CONFIGURED = "ENGINE_NOT_CONFIGURED"
     UPSTREAM_TIMEOUT = "UPSTREAM_TIMEOUT"
     EMPTY_SEARCH_RESULTS = "EMPTY_SEARCH_RESULTS"
+    RESPONSE_TOO_LARGE = "RESPONSE_TOO_LARGE"
 
 
 class ErrorDetail(BaseModel):

{unique_search_proxy_core-2026.44.0.dev6 → unique_search_proxy_core-2026.44.0.dev8}/unique_search_proxy_core/url_safety/__init__.py

@@ -1,6 +1,7 @@
 from unique_search_proxy_core.url_safety.egress import (
     pinned_httpx_get_args,
     safe_pinned_httpx_get,
+    safe_pinned_httpx_stream,
 )
 from unique_search_proxy_core.url_safety.models import (
     BlockedCrawlTarget,
@@ -29,5 +30,6 @@ __all__ = [
     "bypass_crawl_target",
     "pinned_httpx_get_args",
     "safe_pinned_httpx_get",
+    "safe_pinned_httpx_stream",
     "url_safety_settings",
 ]

unique_search_proxy_core-2026.44.0.dev8/unique_search_proxy_core/url_safety/egress.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Mapping
+from contextlib import asynccontextmanager
+from urllib.parse import urlsplit
+
+import httpx
+
+from unique_search_proxy_core.url_safety.models import (
+    BlockedCrawlTarget,
+    CrawlTargetValidationError,
+    ResolvedCrawlTarget,
+    bypass_crawl_target,
+)
+from unique_search_proxy_core.url_safety.redirect import redirect_target_url
+from unique_search_proxy_core.url_safety.resolver import resolve_crawl_target
+
+
+def pinned_httpx_get_args(
+    target: ResolvedCrawlTarget,
+) -> tuple[str, dict[str, str], dict[str, str]]:
+    """Build httpx GET url, headers, and extensions for DNS-pinned egress."""
+    headers: dict[str, str] = {}
+    if target.host_header is not None:
+        headers["Host"] = target.host_header
+
+    extensions: dict[str, str] = {}
+    if target.sni_hostname is not None:
+        extensions["sni_hostname"] = target.sni_hostname
+
+    return target.request_url, headers, extensions
+
+
+def _blocked_redirect(target: ResolvedCrawlTarget, reason: str) -> BlockedCrawlTarget:
+    return BlockedCrawlTarget(
+        hostname=target.hostname or None,
+        category="redirect",
+        reason=reason,
+    )
+
+
+def _next_redirect_url(
+    response: httpx.Response,
+    current_target: ResolvedCrawlTarget,
+) -> str | None:
+    try:
+        next_url = redirect_target_url(response, current_target.normalized_url)
+        if next_url is not None:
+            parsed_next_url = urlsplit(next_url)
+            _ = parsed_next_url.hostname
+            _ = parsed_next_url.port
+    except ValueError as exc:
+        raise CrawlTargetValidationError(
+            [
+                _blocked_redirect(
+                    current_target,
+                    "Redirect target URL is missing or malformed",
+                )
+            ]
+        ) from exc
+    return next_url
+
+
+@asynccontextmanager
+async def safe_pinned_httpx_stream(
+    client: httpx.AsyncClient,
+    target: ResolvedCrawlTarget,
+    *,
+    headers: Mapping[str, str] | None = None,
+    timeout: float | httpx.Timeout,
+    max_redirect_hops: int,
+    enforce_url_safety: bool,
+) -> AsyncIterator[httpx.Response]:
+    """Open a streamed GET, applying the original target's safety mode to redirects.
+
+    Redirect hops are closed without reading their bodies. The final response
+    is yielded unread so the caller decides how much of the body to consume;
+    it is closed when the context exits.
+
+    Raises:
+        CrawlTargetValidationError: When a redirect is malformed, blocked, or
+            exceeds ``max_redirect_hops``.
+    """
+    if max_redirect_hops < 0:
+        raise ValueError("max_redirect_hops must be non-negative")
+
+    current_target = target
+    for redirects_followed in range(max_redirect_hops + 1):
+        request_url, pin_headers, extensions = pinned_httpx_get_args(current_target)
+        request = client.build_request(
+            "GET",
+            request_url,
+            headers={**(headers or {}), **pin_headers},
+            extensions=extensions or None,
+            timeout=timeout,
+        )
+        response = await client.send(request, stream=True, follow_redirects=False)
+        try:
+            next_url = _next_redirect_url(response, current_target)
+            if next_url is None:
+                yield response
+                return
+            if redirects_followed == max_redirect_hops:
+                raise CrawlTargetValidationError(
+                    [
+                        _blocked_redirect(
+                            current_target,
+                            "Maximum redirect hop count exceeded",
+                        )
+                    ]
+                )
+        finally:
+            await response.aclose()
+
+        current_target = (
+            await resolve_crawl_target(next_url)
+            if enforce_url_safety
+            else bypass_crawl_target(next_url)
+        )
+
+    raise AssertionError("redirect loop must return or raise")
+
+
+async def safe_pinned_httpx_get(
+    client: httpx.AsyncClient,
+    target: ResolvedCrawlTarget,
+    *,
+    headers: Mapping[str, str] | None = None,
+    timeout: float | httpx.Timeout,
+    max_redirect_hops: int,
+    enforce_url_safety: bool,
+) -> httpx.Response:
+    """GET a target, applying the original target's safety mode to redirects."""
+    async with safe_pinned_httpx_stream(
+        client,
+        target,
+        headers=headers,
+        timeout=timeout,
+        max_redirect_hops=max_redirect_hops,
+        enforce_url_safety=enforce_url_safety,
+    ) as response:
+        await response.aread()
+    return response
+
+
+__all__ = [
+    "pinned_httpx_get_args",
+    "safe_pinned_httpx_get",
+    "safe_pinned_httpx_stream",
+]

unique_search_proxy_core-2026.44.0.dev6/unique_search_proxy_core/url_safety/egress.py

@@ -1,96 +0,0 @@
-from __future__ import annotations
-
-from collections.abc import Mapping
-from urllib.parse import urlsplit
-
-import httpx
-
-from unique_search_proxy_core.url_safety.models import (
-    BlockedCrawlTarget,
-    CrawlTargetValidationError,
-    ResolvedCrawlTarget,
-    bypass_crawl_target,
-)
-from unique_search_proxy_core.url_safety.redirect import redirect_target_url
-from unique_search_proxy_core.url_safety.resolver import resolve_crawl_target
-
-
-def pinned_httpx_get_args(
-    target: ResolvedCrawlTarget,
-) -> tuple[str, dict[str, str], dict[str, str]]:
-    """Build httpx GET url, headers, and extensions for DNS-pinned egress."""
-    headers: dict[str, str] = {}
-    if target.host_header is not None:
-        headers["Host"] = target.host_header
-
-    extensions: dict[str, str] = {}
-    if target.sni_hostname is not None:
-        extensions["sni_hostname"] = target.sni_hostname
-
-    return target.request_url, headers, extensions
-
-
-async def safe_pinned_httpx_get(
-    client: httpx.AsyncClient,
-    target: ResolvedCrawlTarget,
-    *,
-    headers: Mapping[str, str] | None = None,
-    timeout: float | httpx.Timeout,
-    max_redirect_hops: int,
-    enforce_url_safety: bool,
-) -> httpx.Response:
-    """GET a target, applying the original target's safety mode to redirects."""
-    if max_redirect_hops < 0:
-        raise ValueError("max_redirect_hops must be non-negative")
-
-    current_target = target
-    for redirects_followed in range(max_redirect_hops + 1):
-        request_url, pin_headers, extensions = pinned_httpx_get_args(current_target)
-        response = await client.get(
-            request_url,
-            headers={**(headers or {}), **pin_headers},
-            extensions=extensions or None,
-            timeout=timeout,
-            follow_redirects=False,
-        )
-
-        try:
-            next_url = redirect_target_url(response, current_target.normalized_url)
-            if next_url is not None:
-                parsed_next_url = urlsplit(next_url)
-                _ = parsed_next_url.hostname
-                _ = parsed_next_url.port
-        except ValueError as exc:
-            raise CrawlTargetValidationError(
-                [
-                    BlockedCrawlTarget(
-                        hostname=current_target.hostname or None,
-                        category="redirect",
-                        reason="Redirect target URL is missing or malformed",
-                    )
-                ]
-            ) from exc
-        if next_url is None:
-            return response
-
-        if redirects_followed == max_redirect_hops:
-            raise CrawlTargetValidationError(
-                [
-                    BlockedCrawlTarget(
-                        hostname=current_target.hostname or None,
-                        category="redirect",
-                        reason="Maximum redirect hop count exceeded",
-                    )
-                ]
-            )
-
-        current_target = (
-            await resolve_crawl_target(next_url)
-            if enforce_url_safety
-            else bypass_crawl_target(next_url)
-        )
-
-    raise AssertionError("redirect loop must return or raise")
-
-
-__all__ = ["pinned_httpx_get_args", "safe_pinned_httpx_get"]