unique-search-proxy 2026.44.0.dev8__tar.gz → 2026.44.0.dev10__tar.gz

{unique_search_proxy-2026.44.0.dev8 → unique_search_proxy-2026.44.0.dev10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy
-Version: 2026.44.0.dev8
+Version: 2026.44.0.dev10
 Summary: Web Search Proxy implementation
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>
@@ -21,7 +21,7 @@ Requires-Dist: google-genai>=1.73.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
 Requires-Dist: jinja2>=3.1.6,<4
 Requires-Dist: unique-toolkit[monitoring]>=2026.44.0.dev9,<2026.44.0rc0
-Requires-Dist: unique-search-proxy-core>=2026.44.0.dev7,<2026.44.0rc0
+Requires-Dist: unique-search-proxy-core>=2026.44.0.dev9,<2026.44.0rc0
 Requires-Dist: openai>=1.105.0,<3
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown

{unique_search_proxy-2026.44.0.dev8 → unique_search_proxy-2026.44.0.dev10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy"
-version = "2026.44.0.dev8"
+version = "2026.44.0.dev10"
 description = "Web Search Proxy implementation"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"
@@ -23,7 +23,7 @@ dependencies = [
     "google-auth>=2.43.0,<3",
     "jinja2>=3.1.6,<4",
     "unique-toolkit[monitoring]>=2026.44.0.dev9,<2026.44.0rc0",
-    "unique-search-proxy-core>=2026.44.0.dev7,<2026.44.0rc0",
+    "unique-search-proxy-core>=2026.44.0.dev9,<2026.44.0rc0",
     "openai>=1.105.0,<3",
 ]
 

{unique_search_proxy-2026.44.0.dev8 → unique_search_proxy-2026.44.0.dev10}/unique_search_proxy_client/web/core/crawlers/basic/service.py

@@ -10,6 +10,8 @@ from unique_search_proxy_core.crawlers.basic.processing.policy import (
     ContentTypeHandlerPolicy,
 )
 from unique_search_proxy_core.crawlers.basic.schema import BasicCrawlRequest
+from unique_search_proxy_core.errors import ResponseTooLargeError
+from unique_search_proxy_core.http_client import ByteBudget, read_capped_body
 from unique_search_proxy_core.schema import (
     CrawlUrlResult,
     ProxyErrorCode,
@@ -18,7 +20,7 @@ from unique_search_proxy_core.url_safety import (
     CrawlTargetValidationError,
     ResolvedCrawlTarget,
     bypass_crawl_target,
-    safe_pinned_httpx_get,
+    safe_pinned_httpx_stream,
     url_safety_settings,
 )
 
@@ -37,9 +39,18 @@ from unique_search_proxy_client.web.core.provider_response import (
 )
 from unique_search_proxy_client.web.core.url_safety.gate import AllowedCrawlTarget
 from unique_search_proxy_client.web.monitoring.metrics import record_crawl_blocked
+from unique_search_proxy_client.web.settings.client import http_client_settings
+from unique_search_proxy_client.web.settings.request_limits import (
+    request_limits_settings,
+)
 
 _LOGGER = logging.getLogger(__name__)
 
+# Only encodings whose decoded size read_capped_body can bound in small steps;
+# brotli and zstd reach far higher compression ratios.
+_ACCEPT_ENCODING = "gzip, deflate"
+_COMPRESSING_CONTENT_ENCODINGS = frozenset({"gzip", "deflate"})
+
 
 def _content_type_from_response(response: httpx.Response) -> str | None:
     header = response.headers.get("content-type")
@@ -48,6 +59,25 @@ def _content_type_from_response(response: httpx.Response) -> str | None:
     return str(header).split(";")[0].strip().lower() or None
 
 
+def _unsupported_content_encoding(response: httpx.Response) -> str | None:
+    encodings = [
+        encoding.strip().lower()
+        for encoding in response.headers.get_list(
+            "content-encoding",
+            split_commas=True,
+        )
+    ]
+    for encoding in encodings:
+        if encoding not in {"", "identity", *_COMPRESSING_CONTENT_ENCODINGS}:
+            return encoding
+    # httpx decodes stacked layers in a single step, so a second compressing
+    # layer would inflate past read_capped_body before it can check the size.
+    compressing = [e for e in encodings if e in _COMPRESSING_CONTENT_ENCODINGS]
+    if len(compressing) > 1:
+        return ", ".join(compressing)
+    return None
+
+
 class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
     """Fetch URLs over HTTP and return the response body plus content type."""
 
@@ -79,6 +109,7 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
 
         timeout = request.timeout
         semaphore = asyncio.Semaphore(request.max_concurrent_requests)
+        budget = ByteBudget(request_limits_settings.max_crawl_response_bytes)
         return list(
             await asyncio.gather(
                 *[
@@ -88,6 +119,7 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
                         resolved_target=allowed_target.resolved,
                         timeout=timeout,
                         semaphore=semaphore,
+                        budget=budget,
                         content_type_handlers=request.content_types.to_handlers(),
                     )
                     for allowed_target in allowed_targets
@@ -103,18 +135,51 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
         resolved_target: ResolvedCrawlTarget,
         timeout: int,
         semaphore: asyncio.Semaphore,
+        budget: ByteBudget,
         content_type_handlers: dict[str, ContentTypeHandlerPolicy],
     ) -> CrawlUrlResult:
         async with semaphore:
+            if budget.exhausted:
+                return crawl_upstream_error(
+                    display_url,
+                    "Crawl response budget exhausted before this URL was fetched",
+                    content_type=None,
+                    code=ProxyErrorCode.RESPONSE_TOO_LARGE.value,
+                )
             try:
-                response = await safe_pinned_httpx_get(
+                async with safe_pinned_httpx_stream(
                     client,
                     resolved_target,
-                    headers={"User-Agent": random_user_agent()},
+                    headers={
+                        "User-Agent": random_user_agent(),
+                        "Accept-Encoding": _ACCEPT_ENCODING,
+                    },
                     timeout=Timeout(timeout),
                     max_redirect_hops=url_safety_settings.max_redirect_hops,
                     enforce_url_safety=url_safety_settings.enabled,
-                )
+                ) as response:
+                    content_type = _content_type_from_response(response)
+                    unsupported_encoding = _unsupported_content_encoding(response)
+                    if unsupported_encoding is not None:
+                        _LOGGER.warning(
+                            "Basic crawl rejected content encoding %s",
+                            unsupported_encoding,
+                        )
+                        return crawl_upstream_error(
+                            display_url,
+                            f"Unsupported content encoding: {unsupported_encoding}",
+                            content_type=content_type,
+                            status_code=response.status_code,
+                        )
+                    body = await read_capped_body(
+                        response,
+                        max_bytes=http_client_settings.max_response_bytes,
+                        budget=budget,
+                    )
+                    raw_body = body.decode(
+                        response.encoding or "utf-8",
+                        errors="replace",
+                    )
             except CrawlTargetValidationError as exc:
                 blocked = exc.blocked_targets[0]
                 record_crawl_blocked(blocked.category)
@@ -123,6 +188,14 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
                     blocked.category,
                 )
                 return crawl_forbidden_target(display_url, blocked.reason)
+            except ResponseTooLargeError as exc:
+                _LOGGER.warning("Basic crawl response too large: %s", exc.message)
+                return crawl_upstream_error(
+                    display_url,
+                    exc.message,
+                    content_type=None,
+                    code=ProxyErrorCode.RESPONSE_TOO_LARGE.value,
+                )
             except httpx.TimeoutException as exc:
                 _LOGGER.warning(
                     "Basic crawl timed out error_type=%s",
@@ -147,8 +220,6 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
                     raw=transport_error_raw(exc),
                 )
 
-            content_type = _content_type_from_response(response)
-            raw_body = response.text
             if response.is_error:
                 _LOGGER.warning(
                     "Basic crawl HTTP error http_status=%s",

{unique_search_proxy-2026.44.0.dev8 → unique_search_proxy-2026.44.0.dev10}/unique_search_proxy_client/web/helm/registry.py

@@ -31,6 +31,9 @@ from unique_search_proxy_client.web.settings.providers import (
     tavily_crawl_credentials,
     vertexai_agent_credentials,
 )
+from unique_search_proxy_client.web.settings.request_limits import (
+    request_limits_settings,
+)
 
 
 @dataclass(frozen=True)
@@ -61,6 +64,7 @@ _REGISTERED_INSTANCES: tuple[BaseSettings, ...] = (
     jina_crawl_credentials,
     firecrawl_crawl_credentials,
     http_client_settings,
+    request_limits_settings,
     prometheus_settings,
 )
 

unique_search_proxy-2026.44.0.dev10/unique_search_proxy_client/web/settings/request_limits.py

@@ -0,0 +1,60 @@
+"""Server-side ceilings that bound what a single proxy request may consume."""
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from unique_search_proxy_client.web.helm.metadata import helm_settings
+from unique_search_proxy_client.web.settings.base import get_settings
+
+REQUEST_LIMITS_ENV_PREFIX = "REQUEST_LIMITS_"
+
+
+@helm_settings(
+    title="Request Limits",
+    helm_key="requestLimits",
+    kind="internal",
+    gated=False,
+    egress=None,
+    env_prefix=REQUEST_LIMITS_ENV_PREFIX,
+    sections={
+        "crawl": ["max_crawl_response_kib"],
+    },
+)
+class RequestLimitsSettings(BaseSettings):
+    """Ceilings callers cannot raise, sized against the pod memory limit.
+
+    Byte sizes are in KiB: Helm renders numbers >= 1e6 in scientific notation,
+    which pydantic cannot parse back into an int.
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix=REQUEST_LIMITS_ENV_PREFIX,
+        extra="ignore",
+    )
+
+    max_crawl_response_kib: int = Field(
+        default=10 * 1024,
+        ge=1,
+        description=(
+            "Combined decoded size, in KiB, of all page bodies one crawl request "
+            "may keep. URLs read after the budget is spent fail individually."
+        ),
+    )
+
+    @property
+    def max_crawl_response_bytes(self) -> int:
+        return self.max_crawl_response_kib * 1024
+
+
+def get_request_limits_settings() -> RequestLimitsSettings:
+    return get_settings(RequestLimitsSettings, env_prefix=REQUEST_LIMITS_ENV_PREFIX)
+
+
+request_limits_settings: RequestLimitsSettings = get_request_limits_settings()
+
+__all__ = [
+    "REQUEST_LIMITS_ENV_PREFIX",
+    "RequestLimitsSettings",
+    "get_request_limits_settings",
+    "request_limits_settings",
+]