unique-search-proxy-core 2026.44.0.dev7__tar.gz → 2026.44.0.dev9__tar.gz

{unique_search_proxy_core-2026.44.0.dev7 → unique_search_proxy_core-2026.44.0.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy-core
-Version: 2026.44.0.dev7
+Version: 2026.44.0.dev9
 Summary: Shared Pydantic types for the Unique Search Proxy API
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>

{unique_search_proxy_core-2026.44.0.dev7 → unique_search_proxy_core-2026.44.0.dev9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy-core"
-version = "2026.44.0.dev7"
+version = "2026.44.0.dev9"
 description = "Shared Pydantic types for the Unique Search Proxy API"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"

{unique_search_proxy_core-2026.44.0.dev7 → unique_search_proxy_core-2026.44.0.dev9}/unique_search_proxy_core/http_client/__init__.py

@@ -19,6 +19,7 @@ from unique_search_proxy_core.http_client.credentials import (
 )
 from unique_search_proxy_core.http_client.response_cap import (
     MAX_RESPONSE_BYTES_EXTENSION,
+    ByteBudget,
     install_response_size_cap,
     read_capped_body,
     response_size_cap_hook,
@@ -36,6 +37,7 @@ from unique_search_proxy_core.http_client.settings import (
 
 __all__ = [
     "MAX_RESPONSE_BYTES_EXTENSION",
+    "ByteBudget",
     "DirectRoute",
     "EgressRoute",
     "HttpClientRegistry",

{unique_search_proxy_core-2026.44.0.dev7 → unique_search_proxy_core-2026.44.0.dev9}/unique_search_proxy_core/http_client/response_cap.py

@@ -115,10 +115,45 @@ def install_response_size_cap(
     return client
 
 
+class ByteBudget:
+    """Shared allowance of decoded body bytes across several responses.
+
+    Bounds what one caller keeps in memory at once, e.g. every page of a crawl
+    request, which a per-response limit alone cannot do. Not thread-safe; share
+    it only between tasks of one event loop.
+    """
+
+    def __init__(self, max_bytes: int) -> None:
+        self._max_bytes = max_bytes
+        self._used = 0
+
+    @property
+    def exhausted(self) -> bool:
+        return self._used >= self._max_bytes
+
+    def consume(self, num_bytes: int) -> None:
+        """Reserve bytes, raising once the budget is exceeded.
+
+        Raises:
+            ResponseTooLargeError: When the reservation exceeds the budget.
+        """
+        self._used += num_bytes
+        if self._used > self._max_bytes:
+            raise ResponseTooLargeError(
+                f"Combined upstream responses exceeded the {self._max_bytes} "
+                "byte budget",
+            )
+
+    def release(self, num_bytes: int) -> None:
+        """Return bytes from a body that was discarded."""
+        self._used = max(0, self._used - num_bytes)
+
+
 async def read_capped_body(
     response: httpx.Response,
     *,
     max_bytes: int | None,
+    budget: ByteBudget | None = None,
 ) -> bytes:
     """Read a streamed response body, limiting its decompressed size.
 
@@ -129,25 +164,38 @@ async def read_capped_body(
         response (httpx.Response): Response opened with ``stream=True``.
         max_bytes (int | None): Largest decoded body allowed. ``None`` reads
             the whole body.
+        budget (ByteBudget | None): Shared budget the body is charged against.
+            Bytes are returned to it when the read fails.
 
     Returns:
         bytes: The decoded response body.
 
     Raises:
-        ResponseTooLargeError: When the decoded body exceeds ``max_bytes``.
+        ResponseTooLargeError: When the decoded body exceeds ``max_bytes`` or
+            the shared budget.
     """
     chunks: list[bytes] = []
     received = 0
-    async for chunk in response.aiter_bytes():
-        received += len(chunk)
-        if max_bytes is not None and received > max_bytes:
-            raise ResponseTooLargeError.for_limit(max_bytes)
-        chunks.append(chunk)
+    charged = 0
+    try:
+        async for chunk in response.aiter_bytes():
+            received += len(chunk)
+            if max_bytes is not None and received > max_bytes:
+                raise ResponseTooLargeError.for_limit(max_bytes)
+            if budget is not None:
+                charged += len(chunk)
+                budget.consume(len(chunk))
+            chunks.append(chunk)
+    except BaseException:
+        if budget is not None:
+            budget.release(charged)
+        raise
     return b"".join(chunks)
 
 
 __all__ = [
     "MAX_RESPONSE_BYTES_EXTENSION",
+    "ByteBudget",
     "ResponseHook",
     "install_response_size_cap",
     "read_capped_body",

{unique_search_proxy_core-2026.44.0.dev7 → unique_search_proxy_core-2026.44.0.dev9}/unique_search_proxy_core/url_safety/__init__.py

@@ -1,6 +1,7 @@
 from unique_search_proxy_core.url_safety.egress import (
     pinned_httpx_get_args,
     safe_pinned_httpx_get,
+    safe_pinned_httpx_stream,
 )
 from unique_search_proxy_core.url_safety.models import (
     BlockedCrawlTarget,
@@ -29,5 +30,6 @@ __all__ = [
     "bypass_crawl_target",
     "pinned_httpx_get_args",
     "safe_pinned_httpx_get",
+    "safe_pinned_httpx_stream",
     "url_safety_settings",
 ]

unique_search_proxy_core-2026.44.0.dev9/unique_search_proxy_core/url_safety/egress.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Mapping
+from contextlib import asynccontextmanager
+from urllib.parse import urlsplit
+
+import httpx
+
+from unique_search_proxy_core.url_safety.models import (
+    BlockedCrawlTarget,
+    CrawlTargetValidationError,
+    ResolvedCrawlTarget,
+    bypass_crawl_target,
+)
+from unique_search_proxy_core.url_safety.redirect import redirect_target_url
+from unique_search_proxy_core.url_safety.resolver import resolve_crawl_target
+
+
+def pinned_httpx_get_args(
+    target: ResolvedCrawlTarget,
+) -> tuple[str, dict[str, str], dict[str, str]]:
+    """Build httpx GET url, headers, and extensions for DNS-pinned egress."""
+    headers: dict[str, str] = {}
+    if target.host_header is not None:
+        headers["Host"] = target.host_header
+
+    extensions: dict[str, str] = {}
+    if target.sni_hostname is not None:
+        extensions["sni_hostname"] = target.sni_hostname
+
+    return target.request_url, headers, extensions
+
+
+def _blocked_redirect(target: ResolvedCrawlTarget, reason: str) -> BlockedCrawlTarget:
+    return BlockedCrawlTarget(
+        hostname=target.hostname or None,
+        category="redirect",
+        reason=reason,
+    )
+
+
+def _next_redirect_url(
+    response: httpx.Response,
+    current_target: ResolvedCrawlTarget,
+) -> str | None:
+    try:
+        next_url = redirect_target_url(response, current_target.normalized_url)
+        if next_url is not None:
+            parsed_next_url = urlsplit(next_url)
+            _ = parsed_next_url.hostname
+            _ = parsed_next_url.port
+    except ValueError as exc:
+        raise CrawlTargetValidationError(
+            [
+                _blocked_redirect(
+                    current_target,
+                    "Redirect target URL is missing or malformed",
+                )
+            ]
+        ) from exc
+    return next_url
+
+
+@asynccontextmanager
+async def safe_pinned_httpx_stream(
+    client: httpx.AsyncClient,
+    target: ResolvedCrawlTarget,
+    *,
+    headers: Mapping[str, str] | None = None,
+    timeout: float | httpx.Timeout,
+    max_redirect_hops: int,
+    enforce_url_safety: bool,
+) -> AsyncIterator[httpx.Response]:
+    """Open a streamed GET, applying the original target's safety mode to redirects.
+
+    Redirect hops are closed without reading their bodies. The final response
+    is yielded unread so the caller decides how much of the body to consume;
+    it is closed when the context exits.
+
+    Raises:
+        CrawlTargetValidationError: When a redirect is malformed, blocked, or
+            exceeds ``max_redirect_hops``.
+    """
+    if max_redirect_hops < 0:
+        raise ValueError("max_redirect_hops must be non-negative")
+
+    current_target = target
+    for redirects_followed in range(max_redirect_hops + 1):
+        request_url, pin_headers, extensions = pinned_httpx_get_args(current_target)
+        request = client.build_request(
+            "GET",
+            request_url,
+            headers={**(headers or {}), **pin_headers},
+            extensions=extensions or None,
+            timeout=timeout,
+        )
+        response = await client.send(request, stream=True, follow_redirects=False)
+        try:
+            next_url = _next_redirect_url(response, current_target)
+            if next_url is None:
+                yield response
+                return
+            if redirects_followed == max_redirect_hops:
+                raise CrawlTargetValidationError(
+                    [
+                        _blocked_redirect(
+                            current_target,
+                            "Maximum redirect hop count exceeded",
+                        )
+                    ]
+                )
+        finally:
+            await response.aclose()
+
+        current_target = (
+            await resolve_crawl_target(next_url)
+            if enforce_url_safety
+            else bypass_crawl_target(next_url)
+        )
+
+    raise AssertionError("redirect loop must return or raise")
+
+
+async def safe_pinned_httpx_get(
+    client: httpx.AsyncClient,
+    target: ResolvedCrawlTarget,
+    *,
+    headers: Mapping[str, str] | None = None,
+    timeout: float | httpx.Timeout,
+    max_redirect_hops: int,
+    enforce_url_safety: bool,
+) -> httpx.Response:
+    """GET a target, applying the original target's safety mode to redirects."""
+    async with safe_pinned_httpx_stream(
+        client,
+        target,
+        headers=headers,
+        timeout=timeout,
+        max_redirect_hops=max_redirect_hops,
+        enforce_url_safety=enforce_url_safety,
+    ) as response:
+        await response.aread()
+    return response
+
+
+__all__ = [
+    "pinned_httpx_get_args",
+    "safe_pinned_httpx_get",
+    "safe_pinned_httpx_stream",
+]

unique_search_proxy_core-2026.44.0.dev7/unique_search_proxy_core/url_safety/egress.py

@@ -1,96 +0,0 @@
-from __future__ import annotations
-
-from collections.abc import Mapping
-from urllib.parse import urlsplit
-
-import httpx
-
-from unique_search_proxy_core.url_safety.models import (
-    BlockedCrawlTarget,
-    CrawlTargetValidationError,
-    ResolvedCrawlTarget,
-    bypass_crawl_target,
-)
-from unique_search_proxy_core.url_safety.redirect import redirect_target_url
-from unique_search_proxy_core.url_safety.resolver import resolve_crawl_target
-
-
-def pinned_httpx_get_args(
-    target: ResolvedCrawlTarget,
-) -> tuple[str, dict[str, str], dict[str, str]]:
-    """Build httpx GET url, headers, and extensions for DNS-pinned egress."""
-    headers: dict[str, str] = {}
-    if target.host_header is not None:
-        headers["Host"] = target.host_header
-
-    extensions: dict[str, str] = {}
-    if target.sni_hostname is not None:
-        extensions["sni_hostname"] = target.sni_hostname
-
-    return target.request_url, headers, extensions
-
-
-async def safe_pinned_httpx_get(
-    client: httpx.AsyncClient,
-    target: ResolvedCrawlTarget,
-    *,
-    headers: Mapping[str, str] | None = None,
-    timeout: float | httpx.Timeout,
-    max_redirect_hops: int,
-    enforce_url_safety: bool,
-) -> httpx.Response:
-    """GET a target, applying the original target's safety mode to redirects."""
-    if max_redirect_hops < 0:
-        raise ValueError("max_redirect_hops must be non-negative")
-
-    current_target = target
-    for redirects_followed in range(max_redirect_hops + 1):
-        request_url, pin_headers, extensions = pinned_httpx_get_args(current_target)
-        response = await client.get(
-            request_url,
-            headers={**(headers or {}), **pin_headers},
-            extensions=extensions or None,
-            timeout=timeout,
-            follow_redirects=False,
-        )
-
-        try:
-            next_url = redirect_target_url(response, current_target.normalized_url)
-            if next_url is not None:
-                parsed_next_url = urlsplit(next_url)
-                _ = parsed_next_url.hostname
-                _ = parsed_next_url.port
-        except ValueError as exc:
-            raise CrawlTargetValidationError(
-                [
-                    BlockedCrawlTarget(
-                        hostname=current_target.hostname or None,
-                        category="redirect",
-                        reason="Redirect target URL is missing or malformed",
-                    )
-                ]
-            ) from exc
-        if next_url is None:
-            return response
-
-        if redirects_followed == max_redirect_hops:
-            raise CrawlTargetValidationError(
-                [
-                    BlockedCrawlTarget(
-                        hostname=current_target.hostname or None,
-                        category="redirect",
-                        reason="Maximum redirect hop count exceeded",
-                    )
-                ]
-            )
-
-        current_target = (
-            await resolve_crawl_target(next_url)
-            if enforce_url_safety
-            else bypass_crawl_target(next_url)
-        )
-
-    raise AssertionError("redirect loop must return or raise")
-
-
-__all__ = ["pinned_httpx_get_args", "safe_pinned_httpx_get"]