unique-search-proxy 2026.44.0.dev9__tar.gz → 2026.44.0.dev11__tar.gz

{unique_search_proxy-2026.44.0.dev9 → unique_search_proxy-2026.44.0.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy
-Version: 2026.44.0.dev9
+Version: 2026.44.0.dev11
 Summary: Web Search Proxy implementation
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>
@@ -21,7 +21,7 @@ Requires-Dist: google-genai>=1.73.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
 Requires-Dist: jinja2>=3.1.6,<4
 Requires-Dist: unique-toolkit[monitoring]>=2026.44.0.dev9,<2026.44.0rc0
-Requires-Dist: unique-search-proxy-core>=2026.44.0.dev8,<2026.44.0rc0
+Requires-Dist: unique-search-proxy-core>=2026.44.0.dev9,<2026.44.0rc0
 Requires-Dist: openai>=1.105.0,<3
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown

{unique_search_proxy-2026.44.0.dev9 → unique_search_proxy-2026.44.0.dev11}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy"
-version = "2026.44.0.dev9"
+version = "2026.44.0.dev11"
 description = "Web Search Proxy implementation"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"
@@ -23,7 +23,7 @@ dependencies = [
     "google-auth>=2.43.0,<3",
     "jinja2>=3.1.6,<4",
     "unique-toolkit[monitoring]>=2026.44.0.dev9,<2026.44.0rc0",
-    "unique-search-proxy-core>=2026.44.0.dev8,<2026.44.0rc0",
+    "unique-search-proxy-core>=2026.44.0.dev9,<2026.44.0rc0",
     "openai>=1.105.0,<3",
 ]
 

{unique_search_proxy-2026.44.0.dev9 → unique_search_proxy-2026.44.0.dev11}/unique_search_proxy_client/web/api/v1/crawl.py

@@ -9,6 +9,7 @@ from unique_search_proxy_core.crawlers.config_types import CrawlRequest
 from unique_search_proxy_core.errors import (
     ProxyError,
     UpstreamTimeoutError,
+    ValidationProxyError,
     attach_request_context,
 )
 from unique_search_proxy_core.schema import (
@@ -35,6 +36,9 @@ from unique_search_proxy_client.web.monitoring.metrics import (
     record_crawl_success,
     record_crawl_url_outcomes,
 )
+from unique_search_proxy_client.web.settings.request_limits import (
+    request_limits_settings,
+)
 
 router = APIRouter(tags=["crawl"])
 _LOGGER = logging.getLogger(__name__)
@@ -48,6 +52,21 @@ def _crawl_request_context(exc: ProxyError, *, crawler_id: str) -> ProxyError:
     )
 
 
+def _enforce_url_limits(urls: list[str]) -> None:
+    limits = request_limits_settings
+    if len(urls) > limits.max_urls_per_crawl:
+        raise ValidationProxyError(
+            f"Crawl request lists {len(urls)} URLs; "
+            f"at most {limits.max_urls_per_crawl} are allowed",
+        )
+    for index, url in enumerate(urls):
+        if len(url) > limits.max_url_length:
+            raise ValidationProxyError(
+                f"URL at index {index} is {len(url)} characters long; "
+                f"at most {limits.max_url_length} are allowed",
+            )
+
+
 def _url_outcomes(results: list[CrawlUrlResult]) -> list[tuple[str, str, str]]:
     """Derive ``(outcome, error_code, http_status)`` triples from merged results."""
     outcomes: list[tuple[str, str, str]] = []
@@ -84,6 +103,7 @@ async def crawl(
     )
 
     try:
+        _enforce_url_limits(body.urls)
         async with asyncio.timeout(timeout):
             registry = get_http_client_registry(request.app)
             client = await registry.client_for(get_request_context())

{unique_search_proxy-2026.44.0.dev9 → unique_search_proxy-2026.44.0.dev11}/unique_search_proxy_client/web/core/crawlers/basic/service.py

@@ -11,7 +11,7 @@ from unique_search_proxy_core.crawlers.basic.processing.policy import (
 )
 from unique_search_proxy_core.crawlers.basic.schema import BasicCrawlRequest
 from unique_search_proxy_core.errors import ResponseTooLargeError
-from unique_search_proxy_core.http_client import read_capped_body
+from unique_search_proxy_core.http_client import ByteBudget, read_capped_body
 from unique_search_proxy_core.schema import (
     CrawlUrlResult,
     ProxyErrorCode,
@@ -40,6 +40,9 @@ from unique_search_proxy_client.web.core.provider_response import (
 from unique_search_proxy_client.web.core.url_safety.gate import AllowedCrawlTarget
 from unique_search_proxy_client.web.monitoring.metrics import record_crawl_blocked
 from unique_search_proxy_client.web.settings.client import http_client_settings
+from unique_search_proxy_client.web.settings.request_limits import (
+    request_limits_settings,
+)
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -106,6 +109,7 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
 
         timeout = request.timeout
         semaphore = asyncio.Semaphore(request.max_concurrent_requests)
+        budget = ByteBudget(request_limits_settings.max_crawl_response_bytes)
         return list(
             await asyncio.gather(
                 *[
@@ -115,6 +119,7 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
                         resolved_target=allowed_target.resolved,
                         timeout=timeout,
                         semaphore=semaphore,
+                        budget=budget,
                         content_type_handlers=request.content_types.to_handlers(),
                     )
                     for allowed_target in allowed_targets
@@ -130,9 +135,17 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
         resolved_target: ResolvedCrawlTarget,
         timeout: int,
         semaphore: asyncio.Semaphore,
+        budget: ByteBudget,
         content_type_handlers: dict[str, ContentTypeHandlerPolicy],
     ) -> CrawlUrlResult:
         async with semaphore:
+            if budget.exhausted:
+                return crawl_upstream_error(
+                    display_url,
+                    "Crawl response budget exhausted before this URL was fetched",
+                    content_type=None,
+                    code=ProxyErrorCode.RESPONSE_TOO_LARGE.value,
+                )
             try:
                 async with safe_pinned_httpx_stream(
                     client,
@@ -161,6 +174,7 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
                     body = await read_capped_body(
                         response,
                         max_bytes=http_client_settings.max_response_bytes,
+                        budget=budget,
                     )
                     raw_body = body.decode(
                         response.encoding or "utf-8",

{unique_search_proxy-2026.44.0.dev9 → unique_search_proxy-2026.44.0.dev11}/unique_search_proxy_client/web/helm/registry.py

@@ -31,6 +31,9 @@ from unique_search_proxy_client.web.settings.providers import (
     tavily_crawl_credentials,
     vertexai_agent_credentials,
 )
+from unique_search_proxy_client.web.settings.request_limits import (
+    request_limits_settings,
+)
 
 
 @dataclass(frozen=True)
@@ -61,6 +64,7 @@ _REGISTERED_INSTANCES: tuple[BaseSettings, ...] = (
     jina_crawl_credentials,
     firecrawl_crawl_credentials,
     http_client_settings,
+    request_limits_settings,
     prometheus_settings,
 )
 

unique_search_proxy-2026.44.0.dev11/unique_search_proxy_client/web/settings/request_limits.py

@@ -0,0 +1,79 @@
+"""Server-side ceilings that bound what a single proxy request may consume."""
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from unique_search_proxy_client.web.helm.metadata import helm_settings
+from unique_search_proxy_client.web.settings.base import get_settings
+
+REQUEST_LIMITS_ENV_PREFIX = "REQUEST_LIMITS_"
+
+
+@helm_settings(
+    title="Request Limits",
+    helm_key="requestLimits",
+    kind="internal",
+    gated=False,
+    egress=None,
+    env_prefix=REQUEST_LIMITS_ENV_PREFIX,
+    sections={
+        "crawl": ["max_crawl_response_kib", "max_urls_per_crawl", "max_url_length"],
+    },
+)
+class RequestLimitsSettings(BaseSettings):
+    """Ceilings callers cannot raise, sized against the pod memory limit.
+
+    Byte sizes are in KiB: Helm renders numbers >= 1e6 in scientific notation,
+    which pydantic cannot parse back into an int.
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix=REQUEST_LIMITS_ENV_PREFIX,
+        extra="ignore",
+    )
+
+    max_crawl_response_kib: int = Field(
+        default=10 * 1024,
+        ge=1,
+        description=(
+            "Combined decoded size, in KiB, of all page bodies one crawl request "
+            "may keep. URLs read after the budget is spent fail individually."
+        ),
+    )
+
+    # Matches the search fetch_size ceiling: the web search tool crawls a whole
+    # result page in one request.
+    max_urls_per_crawl: int = Field(
+        default=100,
+        ge=1,
+        description=(
+            "Maximum number of URLs in one crawl request. Requests with more "
+            "are rejected with a 422."
+        ),
+    )
+    max_url_length: int = Field(
+        default=2048,
+        ge=1,
+        description=(
+            "Maximum length of each URL, in characters. Requests containing a "
+            "longer URL are rejected with a 422."
+        ),
+    )
+
+    @property
+    def max_crawl_response_bytes(self) -> int:
+        return self.max_crawl_response_kib * 1024
+
+
+def get_request_limits_settings() -> RequestLimitsSettings:
+    return get_settings(RequestLimitsSettings, env_prefix=REQUEST_LIMITS_ENV_PREFIX)
+
+
+request_limits_settings: RequestLimitsSettings = get_request_limits_settings()
+
+__all__ = [
+    "REQUEST_LIMITS_ENV_PREFIX",
+    "RequestLimitsSettings",
+    "get_request_limits_settings",
+    "request_limits_settings",
+]