unique-search-proxy 2026.44.0.dev10__tar.gz → 2026.44.0.dev12__tar.gz

{unique_search_proxy-2026.44.0.dev10 → unique_search_proxy-2026.44.0.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy
-Version: 2026.44.0.dev10
+Version: 2026.44.0.dev12
 Summary: Web Search Proxy implementation
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>

{unique_search_proxy-2026.44.0.dev10 → unique_search_proxy-2026.44.0.dev12}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy"
-version = "2026.44.0.dev10"
+version = "2026.44.0.dev12"
 description = "Web Search Proxy implementation"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"

{unique_search_proxy-2026.44.0.dev10 → unique_search_proxy-2026.44.0.dev12}/unique_search_proxy_client/web/api/v1/crawl.py

@@ -9,6 +9,7 @@ from unique_search_proxy_core.crawlers.config_types import CrawlRequest
 from unique_search_proxy_core.errors import (
     ProxyError,
     UpstreamTimeoutError,
+    ValidationProxyError,
     attach_request_context,
 )
 from unique_search_proxy_core.schema import (
@@ -22,6 +23,7 @@ from unique_search_proxy_client.web.api.v1.openapi_examples import (
 )
 from unique_search_proxy_client.web.context import get_request_context
 from unique_search_proxy_client.web.core.client import get_http_client_registry
+from unique_search_proxy_client.web.core.crawl_limiter import get_crawl_limiter
 from unique_search_proxy_client.web.core.crawlers.factory import get_crawler_service
 from unique_search_proxy_client.web.core.crawlers.pinned_egress import (
     PinnedEgressCrawler,
@@ -35,6 +37,9 @@ from unique_search_proxy_client.web.monitoring.metrics import (
     record_crawl_success,
     record_crawl_url_outcomes,
 )
+from unique_search_proxy_client.web.settings.request_limits import (
+    request_limits_settings,
+)
 
 router = APIRouter(tags=["crawl"])
 _LOGGER = logging.getLogger(__name__)
@@ -48,6 +53,21 @@ def _crawl_request_context(exc: ProxyError, *, crawler_id: str) -> ProxyError:
     )
 
 
+def _enforce_url_limits(urls: list[str]) -> None:
+    limits = request_limits_settings
+    if len(urls) > limits.max_urls_per_crawl:
+        raise ValidationProxyError(
+            f"Crawl request lists {len(urls)} URLs; "
+            f"at most {limits.max_urls_per_crawl} are allowed",
+        )
+    for index, url in enumerate(urls):
+        if len(url) > limits.max_url_length:
+            raise ValidationProxyError(
+                f"URL at index {index} is {len(url)} characters long; "
+                f"at most {limits.max_url_length} are allowed",
+            )
+
+
 def _url_outcomes(results: list[CrawlUrlResult]) -> list[tuple[str, str, str]]:
     """Derive ``(outcome, error_code, http_status)`` triples from merged results."""
     outcomes: list[tuple[str, str, str]] = []
@@ -74,7 +94,7 @@ async def crawl(
     body: CrawlRequest = Body(openapi_examples=CRAWL_OPENAPI_EXAMPLES),  # type: ignore[valid-type]
 ) -> CrawlResponse:
     crawler_id = body.crawler
-    timeout = body.timeout
+    timeout = min(body.timeout, request_limits_settings.max_timeout_seconds)
     started = time.perf_counter()
     _LOGGER.info(
         "crawl start crawler=%s urls=%d timeout=%ss",
@@ -84,7 +104,8 @@ async def crawl(
     )
 
     try:
-        async with asyncio.timeout(timeout):
+        _enforce_url_limits(body.urls)
+        async with get_crawl_limiter(request.app).slot(), asyncio.timeout(timeout):
             registry = get_http_client_registry(request.app)
             client = await registry.client_for(get_request_context())
             gate = await apply_url_safety_gate(
@@ -122,6 +143,7 @@ async def crawl(
             crawl_body = body.model_copy(
                 update={
                     "urls": [target.display_url for target in gate.allowed_targets],
+                    "timeout": timeout,
                 },
             )
 

{unique_search_proxy-2026.44.0.dev10 → unique_search_proxy-2026.44.0.dev12}/unique_search_proxy_client/web/api/v1/search.py

@@ -31,6 +31,9 @@ from unique_search_proxy_client.web.monitoring.metrics import (
     record_search_error,
     record_search_success,
 )
+from unique_search_proxy_client.web.settings.request_limits import (
+    request_limits_settings,
+)
 
 router = APIRouter(tags=["search"])
 _LOGGER = logging.getLogger(__name__)
@@ -63,7 +66,7 @@ async def search(
 ) -> SearchResponse:
     engine = body.engine
     engine_id = engine.value if hasattr(engine, "value") else str(engine)
-    timeout = body.timeout
+    timeout = min(body.timeout, request_limits_settings.max_timeout_seconds)
     started = time.perf_counter()
     _LOGGER.info("search start engine=%s timeout=%ss", engine_id, timeout)
 

{unique_search_proxy-2026.44.0.dev10 → unique_search_proxy-2026.44.0.dev12}/unique_search_proxy_client/web/app.py

@@ -19,6 +19,7 @@ from unique_search_proxy_client.web.core.agent_engines.bing.client import (
 from unique_search_proxy_client.web.core.client.service import (
     create_http_client_registry,
 )
+from unique_search_proxy_client.web.core.crawl_limiter import CrawlLimiter
 from unique_search_proxy_client.web.core.providers import register_builtin_providers
 from unique_search_proxy_client.web.error_handlers import register_exception_handlers
 from unique_search_proxy_client.web.logging_config import (
@@ -27,6 +28,9 @@ from unique_search_proxy_client.web.logging_config import (
 )
 from unique_search_proxy_client.web.middleware.context import RequestContextMiddleware
 from unique_search_proxy_client.web.monitoring import setup_prometheus
+from unique_search_proxy_client.web.settings.request_limits import (
+    request_limits_settings,
+)
 from unique_search_proxy_client.web.startup_report import (
     log_startup_settings_report,
 )
@@ -61,6 +65,9 @@ async def lifespan(app: FastAPI):
     log_startup_settings_report(_LOGGER)
     registry = await create_http_client_registry()
     app.state.http_client_registry = registry
+    app.state.crawl_limiter = CrawlLimiter(
+        request_limits_settings.max_concurrent_crawls,
+    )
     try:
         await maybe_cleanup_auto_provisioned_bing_agents_on_start(registry)
         yield

unique_search_proxy-2026.44.0.dev12/unique_search_proxy_client/web/core/crawl_limiter.py

@@ -0,0 +1,51 @@
+"""Per-worker limit on crawl requests running at once."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+from unique_search_proxy_core.errors import RateLimitedError
+
+_RETRY_AFTER_SECONDS = 1
+
+
+class CrawlLimiter:
+    """Admits at most ``max_concurrent`` crawls and rejects the rest.
+
+    Rejecting instead of queueing keeps a burst from piling bodies up in one
+    worker; callers retry on the 429.
+    """
+
+    def __init__(self, max_concurrent: int) -> None:
+        self._max_concurrent = max_concurrent
+        self._semaphore = asyncio.Semaphore(max_concurrent)
+
+    @asynccontextmanager
+    async def slot(self) -> AsyncIterator[None]:
+        """Hold one crawl slot for the duration of the block.
+
+        Raises:
+            RateLimitedError: Every slot is taken.
+        """
+        if self._semaphore.locked():
+            raise RateLimitedError(
+                f"Crawl capacity is in use ({self._max_concurrent} concurrent "
+                "crawls per worker)",
+                retry_after_seconds=_RETRY_AFTER_SECONDS,
+            )
+        async with self._semaphore:
+            yield
+
+
+def get_crawl_limiter(app: FastAPI) -> CrawlLimiter:
+    """Return the application-owned crawl limiter."""
+    limiter = getattr(app.state, "crawl_limiter", None)
+    if limiter is None:
+        raise RuntimeError("Crawl limiter is not initialized")
+    return limiter
+
+
+__all__ = ["CrawlLimiter", "get_crawl_limiter"]

{unique_search_proxy-2026.44.0.dev10 → unique_search_proxy-2026.44.0.dev12}/unique_search_proxy_client/web/core/crawlers/basic/service.py

@@ -108,7 +108,12 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
             raise ValueError(msg)
 
         timeout = request.timeout
-        semaphore = asyncio.Semaphore(request.max_concurrent_requests)
+        semaphore = asyncio.Semaphore(
+            min(
+                request.max_concurrent_requests,
+                request_limits_settings.max_concurrent_fetches,
+            ),
+        )
         budget = ByteBudget(request_limits_settings.max_crawl_response_bytes)
         return list(
             await asyncio.gather(

{unique_search_proxy-2026.44.0.dev10 → unique_search_proxy-2026.44.0.dev12}/unique_search_proxy_client/web/settings/request_limits.py

@@ -17,7 +17,14 @@ REQUEST_LIMITS_ENV_PREFIX = "REQUEST_LIMITS_"
     egress=None,
     env_prefix=REQUEST_LIMITS_ENV_PREFIX,
     sections={
-        "crawl": ["max_crawl_response_kib"],
+        "crawl": [
+            "max_crawl_response_kib",
+            "max_urls_per_crawl",
+            "max_url_length",
+            "max_concurrent_fetches",
+            "max_concurrent_crawls",
+        ],
+        "timeouts": ["max_timeout_seconds"],
     },
 )
 class RequestLimitsSettings(BaseSettings):
@@ -41,6 +48,49 @@ class RequestLimitsSettings(BaseSettings):
         ),
     )
 
+    # Matches the search fetch_size ceiling: the web search tool crawls a whole
+    # result page in one request.
+    max_urls_per_crawl: int = Field(
+        default=100,
+        ge=1,
+        description=(
+            "Maximum number of URLs in one crawl request. Requests with more "
+            "are rejected with a 422."
+        ),
+    )
+    max_url_length: int = Field(
+        default=2048,
+        ge=1,
+        description=(
+            "Maximum length of each URL, in characters. Requests containing a "
+            "longer URL are rejected with a 422."
+        ),
+    )
+    max_concurrent_fetches: int = Field(
+        default=10,
+        ge=1,
+        description=(
+            "Ceiling on a basic crawl's maxConcurrentRequests; higher values "
+            "are lowered to this."
+        ),
+    )
+    max_concurrent_crawls: int = Field(
+        default=2,
+        ge=1,
+        description=(
+            "Crawl requests one worker process runs at once. Requests beyond "
+            "this get a 429 instead of queueing."
+        ),
+    )
+    max_timeout_seconds: int = Field(
+        default=120,
+        ge=1,
+        description=(
+            "Ceiling on the timeout of crawl and search requests; longer "
+            "timeouts are lowered to this."
+        ),
+    )
+
     @property
     def max_crawl_response_bytes(self) -> int:
         return self.max_crawl_response_kib * 1024