unique-search-proxy 2026.46.0.dev4__tar.gz → 2026.46.0.dev5__tar.gz

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy
-Version: 2026.46.0.dev4
+Version: 2026.46.0.dev5
 Summary: Web Search Proxy implementation
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>
@@ -240,6 +240,8 @@ Per-URL outcomes — one URL failing does not fail the batch:
 
 Crawler discriminators: `Basic`, `Tavily`, `Jina`, `Firecrawl`.
 
+**Timeout:** with `Basic`, URLs still loading just before `timeout` come back as per-URL `UPSTREAM_TIMEOUT` results next to the finished pages. A busy worker can send the response up to 5 s after `timeout`, so set your HTTP client timeout at least 5 s above it. The other crawlers fail the whole batch with a `504` at `timeout`.
+
 ### 5.5 Errors
 
 Structured envelope on all non-2xx responses:

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev5}/README.md

@@ -212,6 +212,8 @@ Per-URL outcomes — one URL failing does not fail the batch:
 
 Crawler discriminators: `Basic`, `Tavily`, `Jina`, `Firecrawl`.
 
+**Timeout:** with `Basic`, URLs still loading just before `timeout` come back as per-URL `UPSTREAM_TIMEOUT` results next to the finished pages. A busy worker can send the response up to 5 s after `timeout`, so set your HTTP client timeout at least 5 s above it. The other crawlers fail the whole batch with a `504` at `timeout`.
+
 ### 5.5 Errors
 
 Structured envelope on all non-2xx responses:

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy"
-version = "2026.46.0.dev4"
+version = "2026.46.0.dev5"
 description = "Web Search Proxy implementation"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev5}/unique_search_proxy_client/web/api/v1/crawl.py

@@ -45,6 +45,13 @@ from unique_search_proxy_client.web.usage import record_request_usage
 router = APIRouter(tags=["crawl"])
 _LOGGER = logging.getLogger(__name__)
 
+# Callers often use the crawl timeout as their HTTP client timeout. Finishing
+# this much earlier lets the response reach them before they give up.
+_RESPONSE_HEADROOM_SECONDS = 0.5
+# Only catches work that ignores the deadline. A busy worker that cancels pages
+# late must delay the response, not drop the pages that finished.
+_SAFETY_NET_GRACE_SECONDS = 5.0
+
 
 def _crawl_request_context(exc: ProxyError, *, crawler_id: str) -> ProxyError:
     return attach_request_context(
@@ -108,12 +115,18 @@ async def crawl(
 
     try:
         _enforce_url_limits(body.urls)
-        async with get_crawl_limiter(request.app).slot(), asyncio.timeout(timeout):
+        async with (
+            get_crawl_limiter(request.app).slot(),
+            asyncio.timeout(timeout + _SAFETY_NET_GRACE_SECONDS),
+        ):
+            request_deadline = asyncio.get_running_loop().time() + timeout
+            deadline = request_deadline - _RESPONSE_HEADROOM_SECONDS
             registry = get_http_client_registry(request.app)
             client = await registry.client_for(get_request_context())
             gate = await apply_url_safety_gate(
                 body.urls,
                 redirect_http_client=client,
+                deadline=deadline,
             )
             units = len(gate.allowed_targets)
             if not gate.allowed_targets:
@@ -126,7 +139,7 @@ async def crawl(
                 )
                 merged_results = merge_crawl_results(
                     body.urls,
-                    blocked_by_index=gate.blocked_by_index,
+                    uncrawled_by_index=gate.uncrawled_by_index,
                     crawler_results=[],
                 )
                 record_crawl_url_outcomes(
@@ -157,9 +170,11 @@ async def crawl(
                 crawler_results = await crawler.crawl_pinned(
                     crawl_body,
                     gate.allowed_targets,
+                    deadline=deadline,
                 )
             else:
-                crawler_results = await crawler.crawl(crawl_body)
+                async with asyncio.timeout_at(request_deadline):
+                    crawler_results = await crawler.crawl(crawl_body)
         succeeded = True
     except TimeoutError as exc:
         record_crawl_error(
@@ -217,7 +232,7 @@ async def crawl(
     record_crawl_success(crawler_id, len(body.urls), duration)
     merged_results = merge_crawl_results(
         body.urls,
-        blocked_by_index=gate.blocked_by_index,
+        uncrawled_by_index=gate.uncrawled_by_index,
         crawler_results=crawler_results,
     )
     record_crawl_url_outcomes(crawler_id, _url_outcomes(merged_results))

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev5}/unique_search_proxy_client/web/core/crawlers/basic/service.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 import asyncio
 import logging
+from collections.abc import Coroutine
+from typing import Any
 
 import httpx
 from httpx import AsyncClient, Timeout
@@ -33,6 +35,7 @@ from unique_search_proxy_client.web.core.crawlers.basic.user_agent import (
     random_user_agent,
 )
 from unique_search_proxy_client.web.core.provider_response import (
+    crawl_deadline_missed,
     crawl_forbidden_target,
     crawl_upstream_error,
     transport_error_raw,
@@ -97,7 +100,10 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
         self,
         request: BasicCrawlRequest,  # type: ignore[valid-type]
         allowed_targets: list[AllowedCrawlTarget],
+        *,
+        deadline: float | None = None,
     ) -> list[CrawlUrlResult]:
+        """Crawl targets in order; one still running at ``deadline`` fails on its own."""
         client = self._http_client
         if client is None:
             raise RuntimeError("HTTP client is required for Basic crawler")
@@ -115,9 +121,9 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
             ),
         )
         budget = ByteBudget(request_limits_settings.max_crawl_response_bytes)
-        return list(
-            await asyncio.gather(
-                *[
+        results = await asyncio.gather(
+            *[
+                self._crawl_one_before_deadline(
                     self._crawl_one(
                         client,
                         allowed_target.display_url,
@@ -126,11 +132,40 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
                         semaphore=semaphore,
                         budget=budget,
                         content_type_handlers=request.content_types.to_handlers(),
-                    )
-                    for allowed_target in allowed_targets
-                ],
-            ),
+                    ),
+                    deadline=deadline,
+                )
+                for allowed_target in allowed_targets
+            ],
         )
+        missed = sum(result is None for result in results)
+        if missed:
+            _LOGGER.warning(
+                "Basic crawl missed the request deadline timeout=%ss pages=%d",
+                timeout,
+                missed,
+            )
+        return [
+            result
+            if result is not None
+            else crawl_deadline_missed(
+                allowed_target.display_url,
+                f"Crawl did not finish before the {timeout}s request deadline",
+            )
+            for allowed_target, result in zip(allowed_targets, results, strict=True)
+        ]
+
+    async def _crawl_one_before_deadline(
+        self,
+        crawl: Coroutine[Any, Any, CrawlUrlResult],
+        *,
+        deadline: float | None,
+    ) -> CrawlUrlResult | None:
+        try:
+            async with asyncio.timeout_at(deadline):
+                return await crawl
+        except TimeoutError:
+            return None
 
     async def _crawl_one(
         self,

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev5}/unique_search_proxy_client/web/core/crawlers/pinned_egress.py

@@ -16,6 +16,8 @@ class PinnedEgressCrawler(Protocol):
         self,
         request: BaseModel,
         allowed_targets: list[AllowedCrawlTarget],
+        *,
+        deadline: float | None = None,
     ) -> list[CrawlUrlResult]: ...
 
 

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev5}/unique_search_proxy_client/web/core/provider_response.py

@@ -91,6 +91,16 @@ def crawl_forbidden_target(url: str, message: str) -> CrawlUrlResult:
     )
 
 
+def crawl_deadline_missed(url: str, message: str) -> CrawlUrlResult:
+    """Build a per-URL crawl failure for a URL that missed the request deadline."""
+    return crawl_upstream_error(
+        url,
+        message,
+        code=ProxyErrorCode.UPSTREAM_TIMEOUT.value,
+        content_type=None,
+    )
+
+
 def raise_for_upstream_response(
     response: httpx.Response,
     *,

unique_search_proxy-2026.46.0.dev5/unique_search_proxy_client/web/core/url_safety/gate.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from unique_search_proxy_core.schema import CrawlUrlResult
+from unique_search_proxy_core.url_safety import (
+    ResolvedCrawlTarget,
+    UrlSafetyOutcome,
+    UrlSafetyService,
+)
+
+from unique_search_proxy_client.web.core.provider_response import (
+    crawl_deadline_missed,
+    crawl_forbidden_target,
+)
+from unique_search_proxy_client.web.monitoring.metrics import record_crawl_blocked
+
+if TYPE_CHECKING:
+    from httpx import AsyncClient
+
+_LOGGER = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class AllowedCrawlTarget:
+    """User-facing URL paired with the validated resolution for pinned egress."""
+
+    display_url: str
+    resolved: ResolvedCrawlTarget
+
+
+@dataclass(frozen=True)
+class UrlSafetyGateResult:
+    allowed_targets: list[AllowedCrawlTarget]
+    blocked_by_index: dict[int, CrawlUrlResult]
+    timed_out_by_index: dict[int, CrawlUrlResult] = field(default_factory=dict)
+
+    @property
+    def uncrawled_by_index(self) -> dict[int, CrawlUrlResult]:
+        return {**self.blocked_by_index, **self.timed_out_by_index}
+
+
+async def apply_url_safety_gate(
+    urls: list[str],
+    *,
+    redirect_http_client: AsyncClient,
+    deadline: float | None = None,
+) -> UrlSafetyGateResult:
+    """Partition crawl URLs into allowed, blocked and timed-out targets."""
+    allowed_targets: list[AllowedCrawlTarget] = []
+    blocked_by_index: dict[int, CrawlUrlResult] = {}
+    timed_out_by_index: dict[int, CrawlUrlResult] = {}
+
+    for index, url in enumerate(urls):
+        outcome = await _validate_before_deadline(
+            url,
+            redirect_http_client=redirect_http_client,
+            deadline=deadline,
+        )
+        if outcome is None:
+            timed_out_by_index = {
+                unchecked_index: crawl_deadline_missed(
+                    unchecked_url.strip(),
+                    "URL safety check did not finish before the crawl deadline",
+                )
+                for unchecked_index, unchecked_url in enumerate(
+                    urls[index:],
+                    start=index,
+                )
+            }
+            _LOGGER.warning(
+                "URL safety checks missed the crawl deadline unchecked=%d",
+                len(timed_out_by_index),
+            )
+            break
+
+        if outcome.blocked is not None:
+            record_crawl_blocked(outcome.blocked.category)
+            blocked_by_index[index] = crawl_forbidden_target(
+                outcome.url.strip(),
+                outcome.blocked.reason,
+            )
+            continue
+
+        if outcome.resolved is None:
+            msg = "URL safety allowed a crawl target without resolved metadata"
+            raise RuntimeError(msg)
+
+        allowed_targets.append(
+            AllowedCrawlTarget(
+                display_url=outcome.url.strip(),
+                resolved=outcome.resolved,
+            ),
+        )
+
+    return UrlSafetyGateResult(
+        allowed_targets=allowed_targets,
+        blocked_by_index=blocked_by_index,
+        timed_out_by_index=timed_out_by_index,
+    )
+
+
+async def _validate_before_deadline(
+    url: str,
+    *,
+    redirect_http_client: AsyncClient,
+    deadline: float | None,
+) -> UrlSafetyOutcome | None:
+    """Return the URL's safety outcome, or ``None`` when the deadline passes first."""
+    try:
+        async with asyncio.timeout_at(deadline):
+            [outcome] = await UrlSafetyService.validate_urls_individually(
+                [url],
+                redirect_http_client=redirect_http_client,
+            )
+    except TimeoutError:
+        return None
+    return outcome
+
+
+def merge_crawl_results(
+    urls: list[str],
+    *,
+    uncrawled_by_index: dict[int, CrawlUrlResult],
+    crawler_results: list[CrawlUrlResult],
+) -> list[CrawlUrlResult]:
+    """Merge the gate's results for uncrawled URLs with crawler outcomes in order."""
+    merged: list[CrawlUrlResult] = []
+    crawler_index = 0
+    for index, _url in enumerate(urls):
+        uncrawled = uncrawled_by_index.get(index)
+        if uncrawled is not None:
+            merged.append(uncrawled)
+            continue
+
+        merged.append(crawler_results[crawler_index])
+        crawler_index += 1
+
+    return merged

unique_search_proxy-2026.46.0.dev4/unique_search_proxy_client/web/core/url_safety/gate.py

@@ -1,87 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import TYPE_CHECKING
-
-from unique_search_proxy_core.schema import CrawlUrlResult
-from unique_search_proxy_core.url_safety import ResolvedCrawlTarget, UrlSafetyService
-
-from unique_search_proxy_client.web.core.provider_response import crawl_forbidden_target
-from unique_search_proxy_client.web.monitoring.metrics import record_crawl_blocked
-
-if TYPE_CHECKING:
-    from httpx import AsyncClient
-
-
-@dataclass(frozen=True)
-class AllowedCrawlTarget:
-    """User-facing URL paired with the validated resolution for pinned egress."""
-
-    display_url: str
-    resolved: ResolvedCrawlTarget
-
-
-@dataclass(frozen=True)
-class UrlSafetyGateResult:
-    allowed_targets: list[AllowedCrawlTarget]
-    blocked_by_index: dict[int, CrawlUrlResult]
-
-
-async def apply_url_safety_gate(
-    urls: list[str],
-    *,
-    redirect_http_client: AsyncClient,
-) -> UrlSafetyGateResult:
-    """Validate crawl URLs and partition them into allowed vs blocked targets."""
-    outcomes = await UrlSafetyService.validate_urls_individually(
-        urls,
-        redirect_http_client=redirect_http_client,
-    )
-    allowed_targets: list[AllowedCrawlTarget] = []
-    blocked_by_index: dict[int, CrawlUrlResult] = {}
-
-    for index, outcome in enumerate(outcomes):
-        if outcome.blocked is not None:
-            record_crawl_blocked(outcome.blocked.category)
-            blocked_by_index[index] = crawl_forbidden_target(
-                outcome.url.strip(),
-                outcome.blocked.reason,
-            )
-            continue
-
-        if outcome.resolved is None:
-            msg = "URL safety allowed a crawl target without resolved metadata"
-            raise RuntimeError(msg)
-
-        allowed_targets.append(
-            AllowedCrawlTarget(
-                display_url=outcome.url.strip(),
-                resolved=outcome.resolved,
-            ),
-        )
-
-    return UrlSafetyGateResult(
-        allowed_targets=allowed_targets,
-        blocked_by_index=blocked_by_index,
-    )
-
-
-def merge_crawl_results(
-    urls: list[str],
-    *,
-    blocked_by_index: dict[int, CrawlUrlResult],
-    crawler_results: list[CrawlUrlResult],
-) -> list[CrawlUrlResult]:
-    """Merge per-URL blocked results with crawler outcomes in request order."""
-    merged: list[CrawlUrlResult] = []
-    crawler_index = 0
-    for index, _url in enumerate(urls):
-        blocked = blocked_by_index.get(index)
-        if blocked is not None:
-            merged.append(blocked)
-            continue
-
-        merged.append(crawler_results[crawler_index])
-        crawler_index += 1
-
-    return merged