unique-search-proxy 2026.44.0.dev10__tar.gz → 2026.44.0.dev11__tar.gz

{unique_search_proxy-2026.44.0.dev10 → unique_search_proxy-2026.44.0.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy
-Version: 2026.44.0.dev10
+Version: 2026.44.0.dev11
 Summary: Web Search Proxy implementation
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>

{unique_search_proxy-2026.44.0.dev10 → unique_search_proxy-2026.44.0.dev11}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy"
-version = "2026.44.0.dev10"
+version = "2026.44.0.dev11"
 description = "Web Search Proxy implementation"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"

{unique_search_proxy-2026.44.0.dev10 → unique_search_proxy-2026.44.0.dev11}/unique_search_proxy_client/web/api/v1/crawl.py

@@ -9,6 +9,7 @@ from unique_search_proxy_core.crawlers.config_types import CrawlRequest
 from unique_search_proxy_core.errors import (
     ProxyError,
     UpstreamTimeoutError,
+    ValidationProxyError,
     attach_request_context,
 )
 from unique_search_proxy_core.schema import (
@@ -35,6 +36,9 @@ from unique_search_proxy_client.web.monitoring.metrics import (
     record_crawl_success,
     record_crawl_url_outcomes,
 )
+from unique_search_proxy_client.web.settings.request_limits import (
+    request_limits_settings,
+)
 
 router = APIRouter(tags=["crawl"])
 _LOGGER = logging.getLogger(__name__)
@@ -48,6 +52,21 @@ def _crawl_request_context(exc: ProxyError, *, crawler_id: str) -> ProxyError:
     )
 
 
+def _enforce_url_limits(urls: list[str]) -> None:
+    limits = request_limits_settings
+    if len(urls) > limits.max_urls_per_crawl:
+        raise ValidationProxyError(
+            f"Crawl request lists {len(urls)} URLs; "
+            f"at most {limits.max_urls_per_crawl} are allowed",
+        )
+    for index, url in enumerate(urls):
+        if len(url) > limits.max_url_length:
+            raise ValidationProxyError(
+                f"URL at index {index} is {len(url)} characters long; "
+                f"at most {limits.max_url_length} are allowed",
+            )
+
+
 def _url_outcomes(results: list[CrawlUrlResult]) -> list[tuple[str, str, str]]:
     """Derive ``(outcome, error_code, http_status)`` triples from merged results."""
     outcomes: list[tuple[str, str, str]] = []
@@ -84,6 +103,7 @@ async def crawl(
     )
 
     try:
+        _enforce_url_limits(body.urls)
         async with asyncio.timeout(timeout):
             registry = get_http_client_registry(request.app)
             client = await registry.client_for(get_request_context())

{unique_search_proxy-2026.44.0.dev10 → unique_search_proxy-2026.44.0.dev11}/unique_search_proxy_client/web/settings/request_limits.py

@@ -17,7 +17,7 @@ REQUEST_LIMITS_ENV_PREFIX = "REQUEST_LIMITS_"
     egress=None,
     env_prefix=REQUEST_LIMITS_ENV_PREFIX,
     sections={
-        "crawl": ["max_crawl_response_kib"],
+        "crawl": ["max_crawl_response_kib", "max_urls_per_crawl", "max_url_length"],
     },
 )
 class RequestLimitsSettings(BaseSettings):
@@ -41,6 +41,25 @@ class RequestLimitsSettings(BaseSettings):
         ),
     )
 
+    # Matches the search fetch_size ceiling: the web search tool crawls a whole
+    # result page in one request.
+    max_urls_per_crawl: int = Field(
+        default=100,
+        ge=1,
+        description=(
+            "Maximum number of URLs in one crawl request. Requests with more "
+            "are rejected with a 422."
+        ),
+    )
+    max_url_length: int = Field(
+        default=2048,
+        ge=1,
+        description=(
+            "Maximum length of each URL, in characters. Requests containing a "
+            "longer URL are rejected with a 422."
+        ),
+    )
+
     @property
     def max_crawl_response_bytes(self) -> int:
         return self.max_crawl_response_kib * 1024