unique-search-proxy 2026.46.0.dev4__tar.gz → 2026.46.0.dev6__tar.gz

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy
-Version: 2026.46.0.dev4
+Version: 2026.46.0.dev6
 Summary: Web Search Proxy implementation
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>
@@ -20,8 +20,8 @@ Requires-Dist: certifi>=2025.11.12,<2027
 Requires-Dist: google-genai>=1.73.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
 Requires-Dist: jinja2>=3.1.6,<4
-Requires-Dist: unique-toolkit[monitoring]>=2026.46.0.dev4,<2026.46.0rc0
-Requires-Dist: unique-search-proxy-core>=2026.46.0.dev1,<2026.46.0rc0
+Requires-Dist: unique-toolkit[monitoring]>=2026.46.0.dev10,<2026.46.0rc0
+Requires-Dist: unique-search-proxy-core>=2026.46.0.dev2,<2026.46.0rc0
 Requires-Dist: openai>=1.105.0,<3
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -240,6 +240,8 @@ Per-URL outcomes — one URL failing does not fail the batch:
 
 Crawler discriminators: `Basic`, `Tavily`, `Jina`, `Firecrawl`.
 
+**Timeout:** with `Basic`, URLs still loading just before `timeout` come back as per-URL `UPSTREAM_TIMEOUT` results next to the finished pages. A busy worker can send the response up to 5 s after `timeout`, so set your HTTP client timeout at least 5 s above it. The other crawlers fail the whole batch with a `504` at `timeout`.
+
 ### 5.5 Errors
 
 Structured envelope on all non-2xx responses:

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev6}/README.md

@@ -212,6 +212,8 @@ Per-URL outcomes — one URL failing does not fail the batch:
 
 Crawler discriminators: `Basic`, `Tavily`, `Jina`, `Firecrawl`.
 
+**Timeout:** with `Basic`, URLs still loading just before `timeout` come back as per-URL `UPSTREAM_TIMEOUT` results next to the finished pages. A busy worker can send the response up to 5 s after `timeout`, so set your HTTP client timeout at least 5 s above it. The other crawlers fail the whole batch with a `504` at `timeout`.
+
 ### 5.5 Errors
 
 Structured envelope on all non-2xx responses:

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy"
-version = "2026.46.0.dev4"
+version = "2026.46.0.dev6"
 description = "Web Search Proxy implementation"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"
@@ -22,8 +22,8 @@ dependencies = [
     "google-genai>=1.73.0,<2",
     "google-auth>=2.43.0,<3",
     "jinja2>=3.1.6,<4",
-    "unique-toolkit[monitoring]>=2026.46.0.dev4,<2026.46.0rc0",
-    "unique-search-proxy-core>=2026.46.0.dev1,<2026.46.0rc0",
+    "unique-toolkit[monitoring]>=2026.46.0.dev10,<2026.46.0rc0",
+    "unique-search-proxy-core>=2026.46.0.dev2,<2026.46.0rc0",
     "openai>=1.105.0,<3",
 ]
 

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev6}/unique_search_proxy_client/web/api/v1/crawl.py

@@ -45,6 +45,13 @@ from unique_search_proxy_client.web.usage import record_request_usage
 router = APIRouter(tags=["crawl"])
 _LOGGER = logging.getLogger(__name__)
 
+# Callers often use the crawl timeout as their HTTP client timeout. Finishing
+# this much earlier lets the response reach them before they give up.
+_RESPONSE_HEADROOM_SECONDS = 0.5
+# Only catches work that ignores the deadline. A busy worker that cancels pages
+# late must delay the response, not drop the pages that finished.
+_SAFETY_NET_GRACE_SECONDS = 5.0
+
 
 def _crawl_request_context(exc: ProxyError, *, crawler_id: str) -> ProxyError:
     return attach_request_context(
@@ -108,12 +115,18 @@ async def crawl(
 
     try:
         _enforce_url_limits(body.urls)
-        async with get_crawl_limiter(request.app).slot(), asyncio.timeout(timeout):
+        async with (
+            get_crawl_limiter(request.app).slot(),
+            asyncio.timeout(timeout + _SAFETY_NET_GRACE_SECONDS),
+        ):
+            request_deadline = asyncio.get_running_loop().time() + timeout
+            deadline = request_deadline - _RESPONSE_HEADROOM_SECONDS
             registry = get_http_client_registry(request.app)
             client = await registry.client_for(get_request_context())
             gate = await apply_url_safety_gate(
                 body.urls,
                 redirect_http_client=client,
+                deadline=deadline,
             )
             units = len(gate.allowed_targets)
             if not gate.allowed_targets:
@@ -126,7 +139,7 @@ async def crawl(
                 )
                 merged_results = merge_crawl_results(
                     body.urls,
-                    blocked_by_index=gate.blocked_by_index,
+                    uncrawled_by_index=gate.uncrawled_by_index,
                     crawler_results=[],
                 )
                 record_crawl_url_outcomes(
@@ -157,9 +170,11 @@ async def crawl(
                 crawler_results = await crawler.crawl_pinned(
                     crawl_body,
                     gate.allowed_targets,
+                    deadline=deadline,
                 )
             else:
-                crawler_results = await crawler.crawl(crawl_body)
+                async with asyncio.timeout_at(request_deadline):
+                    crawler_results = await crawler.crawl(crawl_body)
         succeeded = True
     except TimeoutError as exc:
         record_crawl_error(
@@ -217,7 +232,7 @@ async def crawl(
     record_crawl_success(crawler_id, len(body.urls), duration)
     merged_results = merge_crawl_results(
         body.urls,
-        blocked_by_index=gate.blocked_by_index,
+        uncrawled_by_index=gate.uncrawled_by_index,
         crawler_results=crawler_results,
     )
     record_crawl_url_outcomes(crawler_id, _url_outcomes(merged_results))

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev6}/unique_search_proxy_client/web/core/crawlers/basic/service.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 import asyncio
 import logging
+from collections.abc import Coroutine
+from typing import Any
 
 import httpx
 from httpx import AsyncClient, Timeout
@@ -33,6 +35,7 @@ from unique_search_proxy_client.web.core.crawlers.basic.user_agent import (
     random_user_agent,
 )
 from unique_search_proxy_client.web.core.provider_response import (
+    crawl_deadline_missed,
     crawl_forbidden_target,
     crawl_upstream_error,
     transport_error_raw,
@@ -97,7 +100,10 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
         self,
         request: BasicCrawlRequest,  # type: ignore[valid-type]
         allowed_targets: list[AllowedCrawlTarget],
+        *,
+        deadline: float | None = None,
     ) -> list[CrawlUrlResult]:
+        """Crawl targets in order; one still running at ``deadline`` fails on its own."""
         client = self._http_client
         if client is None:
             raise RuntimeError("HTTP client is required for Basic crawler")
@@ -115,9 +121,9 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
             ),
         )
         budget = ByteBudget(request_limits_settings.max_crawl_response_bytes)
-        return list(
-            await asyncio.gather(
-                *[
+        results = await asyncio.gather(
+            *[
+                self._crawl_one_before_deadline(
                     self._crawl_one(
                         client,
                         allowed_target.display_url,
@@ -126,11 +132,40 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
                         semaphore=semaphore,
                         budget=budget,
                         content_type_handlers=request.content_types.to_handlers(),
-                    )
-                    for allowed_target in allowed_targets
-                ],
-            ),
+                    ),
+                    deadline=deadline,
+                )
+                for allowed_target in allowed_targets
+            ],
         )
+        missed = sum(result is None for result in results)
+        if missed:
+            _LOGGER.warning(
+                "Basic crawl missed the request deadline timeout=%ss pages=%d",
+                timeout,
+                missed,
+            )
+        return [
+            result
+            if result is not None
+            else crawl_deadline_missed(
+                allowed_target.display_url,
+                f"Crawl did not finish before the {timeout}s request deadline",
+            )
+            for allowed_target, result in zip(allowed_targets, results, strict=True)
+        ]
+
+    async def _crawl_one_before_deadline(
+        self,
+        crawl: Coroutine[Any, Any, CrawlUrlResult],
+        *,
+        deadline: float | None,
+    ) -> CrawlUrlResult | None:
+        try:
+            async with asyncio.timeout_at(deadline):
+                return await crawl
+        except TimeoutError:
+            return None
 
     async def _crawl_one(
         self,

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev6}/unique_search_proxy_client/web/core/crawlers/pinned_egress.py

@@ -16,6 +16,8 @@ class PinnedEgressCrawler(Protocol):
         self,
         request: BaseModel,
         allowed_targets: list[AllowedCrawlTarget],
+        *,
+        deadline: float | None = None,
     ) -> list[CrawlUrlResult]: ...
 
 

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev6}/unique_search_proxy_client/web/core/provider_response.py

@@ -91,6 +91,16 @@ def crawl_forbidden_target(url: str, message: str) -> CrawlUrlResult:
     )
 
 
+def crawl_deadline_missed(url: str, message: str) -> CrawlUrlResult:
+    """Build a per-URL crawl failure for a URL that missed the request deadline."""
+    return crawl_upstream_error(
+        url,
+        message,
+        code=ProxyErrorCode.UPSTREAM_TIMEOUT.value,
+        content_type=None,
+    )
+
+
 def raise_for_upstream_response(
     response: httpx.Response,
     *,

unique_search_proxy-2026.46.0.dev6/unique_search_proxy_client/web/core/url_safety/gate.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from unique_search_proxy_core.schema import CrawlUrlResult
+from unique_search_proxy_core.url_safety import (
+    ResolvedCrawlTarget,
+    UrlSafetyOutcome,
+    UrlSafetyService,
+)
+
+from unique_search_proxy_client.web.core.provider_response import (
+    crawl_deadline_missed,
+    crawl_forbidden_target,
+)
+from unique_search_proxy_client.web.monitoring.metrics import (
+    record_crawl_blocked,
+    record_url_safety_check,
+)
+from unique_search_proxy_client.web.settings.request_limits import (
+    request_limits_settings,
+)
+
+if TYPE_CHECKING:
+    from httpx import AsyncClient
+
+_LOGGER = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class AllowedCrawlTarget:
+    """User-facing URL paired with the validated resolution for pinned egress."""
+
+    display_url: str
+    resolved: ResolvedCrawlTarget
+
+
+@dataclass(frozen=True)
+class UrlSafetyGateResult:
+    allowed_targets: list[AllowedCrawlTarget]
+    blocked_by_index: dict[int, CrawlUrlResult]
+    timed_out_by_index: dict[int, CrawlUrlResult] = field(default_factory=dict)
+
+    @property
+    def uncrawled_by_index(self) -> dict[int, CrawlUrlResult]:
+        return {**self.blocked_by_index, **self.timed_out_by_index}
+
+
+async def apply_url_safety_gate(
+    urls: list[str],
+    *,
+    redirect_http_client: AsyncClient,
+    deadline: float | None = None,
+) -> UrlSafetyGateResult:
+    """Partition crawl URLs into allowed, blocked and timed-out targets."""
+    check_slots = asyncio.Semaphore(request_limits_settings.max_concurrent_url_checks)
+    outcomes = await asyncio.gather(
+        *[
+            _validate_before_deadline(
+                url,
+                redirect_http_client=redirect_http_client,
+                check_slots=check_slots,
+                deadline=deadline,
+            )
+            for url in urls
+        ],
+    )
+
+    allowed_targets: list[AllowedCrawlTarget] = []
+    blocked_by_index: dict[int, CrawlUrlResult] = {}
+    timed_out_by_index: dict[int, CrawlUrlResult] = {}
+
+    for index, (url, outcome) in enumerate(zip(urls, outcomes, strict=True)):
+        if outcome is None:
+            timed_out_by_index[index] = crawl_deadline_missed(
+                url.strip(),
+                "URL safety check did not finish before the crawl deadline",
+            )
+            continue
+
+        if outcome.blocked is not None:
+            record_crawl_blocked(outcome.blocked.category)
+            blocked_by_index[index] = crawl_forbidden_target(
+                outcome.url.strip(),
+                outcome.blocked.reason,
+            )
+            continue
+
+        if outcome.resolved is None:
+            msg = "URL safety allowed a crawl target without resolved metadata"
+            raise RuntimeError(msg)
+
+        allowed_targets.append(
+            AllowedCrawlTarget(
+                display_url=outcome.url.strip(),
+                resolved=outcome.resolved,
+            ),
+        )
+
+    if timed_out_by_index:
+        _LOGGER.warning(
+            "URL safety checks missed the crawl deadline unchecked=%d",
+            len(timed_out_by_index),
+        )
+
+    return UrlSafetyGateResult(
+        allowed_targets=allowed_targets,
+        blocked_by_index=blocked_by_index,
+        timed_out_by_index=timed_out_by_index,
+    )
+
+
+async def _validate_before_deadline(
+    url: str,
+    *,
+    redirect_http_client: AsyncClient,
+    check_slots: asyncio.Semaphore,
+    deadline: float | None,
+) -> UrlSafetyOutcome | None:
+    """Return the URL's safety outcome, or ``None`` when the deadline passes first."""
+    try:
+        async with asyncio.timeout_at(deadline), check_slots:
+            return await _validate_and_record(
+                url,
+                redirect_http_client=redirect_http_client,
+            )
+    except TimeoutError:
+        return None
+
+
+async def _validate_and_record(
+    url: str,
+    *,
+    redirect_http_client: AsyncClient,
+) -> UrlSafetyOutcome:
+    """Validate one URL and record how long the check took, by outcome."""
+    started = time.perf_counter()
+    outcome_label = "unfinished"
+    try:
+        [outcome] = await UrlSafetyService.validate_urls_individually(
+            [url],
+            redirect_http_client=redirect_http_client,
+        )
+        outcome_label = "allowed" if outcome.blocked is None else "blocked"
+        return outcome
+    finally:
+        record_url_safety_check(outcome_label, time.perf_counter() - started)
+
+
+def merge_crawl_results(
+    urls: list[str],
+    *,
+    uncrawled_by_index: dict[int, CrawlUrlResult],
+    crawler_results: list[CrawlUrlResult],
+) -> list[CrawlUrlResult]:
+    """Merge the gate's results for uncrawled URLs with crawler outcomes in order."""
+    merged: list[CrawlUrlResult] = []
+    crawler_index = 0
+    for index, _url in enumerate(urls):
+        uncrawled = uncrawled_by_index.get(index)
+        if uncrawled is not None:
+            merged.append(uncrawled)
+            continue
+
+        merged.append(crawler_results[crawler_index])
+        crawler_index += 1
+
+    return merged

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev6}/unique_search_proxy_client/web/monitoring/metrics.py

@@ -40,6 +40,9 @@ def _exponential_buckets(start: float, ceiling: float, count: int) -> tuple[floa
 _SEARCH_LATENCY_BUCKETS = _linear_buckets(ceiling=5.0, steps=20)
 _CRAWL_LATENCY_BUCKETS = _linear_buckets(ceiling=60.0, steps=20)
 _AGENT_SEARCH_LATENCY_BUCKETS = _linear_buckets(ceiling=120.0, steps=20)
+# Half-second steps up to 10s, so checks that hit the per-hop redirect timeout
+# (5s by default) show up as their own bucket.
+_URL_SAFETY_CHECK_LATENCY_BUCKETS = _linear_buckets(ceiling=10.0, steps=20)
 
 # One histogram covers every endpoint, from sub-second /v1/search to /v1/agent-search
 # runs approaching 120s — exponential buckets keep resolution useful at both ends.
@@ -94,6 +97,12 @@ crawl_blocked_total = m.counter(
     "Crawl targets blocked by URL safety policy",
     ["reason_category"],
 )
+url_safety_check_duration_seconds = m.histogram(
+    "url_safety_check_duration_seconds",
+    "Time to check one crawl URL (redirect probes and DNS), by outcome",
+    ["outcome"],
+    buckets=_URL_SAFETY_CHECK_LATENCY_BUCKETS,
+)
 url_safety_application_checks_enabled = m.gauge(
     "url_safety_application_checks_enabled",
     "Whether application-level URL safety checks are enabled (1) or delegated "
@@ -182,6 +191,11 @@ def record_crawl_blocked(reason_category: str, count: int = 1) -> None:
     crawl_blocked_total.labels(reason_category=reason_category).inc(count)
 
 
+def record_url_safety_check(outcome: str, duration_seconds: float) -> None:
+    """Record one URL check; ``outcome`` is allowed, blocked or unfinished."""
+    url_safety_check_duration_seconds.labels(outcome=outcome).observe(duration_seconds)
+
+
 def set_url_safety_application_checks_enabled(enabled: bool) -> None:
     url_safety_application_checks_enabled.set(int(enabled))
 

{unique_search_proxy-2026.46.0.dev4 → unique_search_proxy-2026.46.0.dev6}/unique_search_proxy_client/web/settings/request_limits.py

@@ -23,6 +23,7 @@ REQUEST_LIMITS_ENV_PREFIX = "REQUEST_LIMITS_"
             "max_url_length",
             "max_concurrent_fetches",
             "max_concurrent_crawls",
+            "max_concurrent_url_checks",
         ],
         "timeouts": ["max_timeout_seconds"],
     },
@@ -82,6 +83,14 @@ class RequestLimitsSettings(BaseSettings):
             "this get a 429 instead of queueing."
         ),
     )
+    max_concurrent_url_checks: int = Field(
+        default=3,
+        ge=1,
+        description=(
+            "URL safety checks one crawl request runs at once before fetching. "
+            "Each check probes the URL's redirects with HEAD requests."
+        ),
+    )
     max_timeout_seconds: int = Field(
         default=120,
         ge=1,

unique_search_proxy-2026.46.0.dev4/unique_search_proxy_client/web/core/url_safety/gate.py

@@ -1,87 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import TYPE_CHECKING
-
-from unique_search_proxy_core.schema import CrawlUrlResult
-from unique_search_proxy_core.url_safety import ResolvedCrawlTarget, UrlSafetyService
-
-from unique_search_proxy_client.web.core.provider_response import crawl_forbidden_target
-from unique_search_proxy_client.web.monitoring.metrics import record_crawl_blocked
-
-if TYPE_CHECKING:
-    from httpx import AsyncClient
-
-
-@dataclass(frozen=True)
-class AllowedCrawlTarget:
-    """User-facing URL paired with the validated resolution for pinned egress."""
-
-    display_url: str
-    resolved: ResolvedCrawlTarget
-
-
-@dataclass(frozen=True)
-class UrlSafetyGateResult:
-    allowed_targets: list[AllowedCrawlTarget]
-    blocked_by_index: dict[int, CrawlUrlResult]
-
-
-async def apply_url_safety_gate(
-    urls: list[str],
-    *,
-    redirect_http_client: AsyncClient,
-) -> UrlSafetyGateResult:
-    """Validate crawl URLs and partition them into allowed vs blocked targets."""
-    outcomes = await UrlSafetyService.validate_urls_individually(
-        urls,
-        redirect_http_client=redirect_http_client,
-    )
-    allowed_targets: list[AllowedCrawlTarget] = []
-    blocked_by_index: dict[int, CrawlUrlResult] = {}
-
-    for index, outcome in enumerate(outcomes):
-        if outcome.blocked is not None:
-            record_crawl_blocked(outcome.blocked.category)
-            blocked_by_index[index] = crawl_forbidden_target(
-                outcome.url.strip(),
-                outcome.blocked.reason,
-            )
-            continue
-
-        if outcome.resolved is None:
-            msg = "URL safety allowed a crawl target without resolved metadata"
-            raise RuntimeError(msg)
-
-        allowed_targets.append(
-            AllowedCrawlTarget(
-                display_url=outcome.url.strip(),
-                resolved=outcome.resolved,
-            ),
-        )
-
-    return UrlSafetyGateResult(
-        allowed_targets=allowed_targets,
-        blocked_by_index=blocked_by_index,
-    )
-
-
-def merge_crawl_results(
-    urls: list[str],
-    *,
-    blocked_by_index: dict[int, CrawlUrlResult],
-    crawler_results: list[CrawlUrlResult],
-) -> list[CrawlUrlResult]:
-    """Merge per-URL blocked results with crawler outcomes in request order."""
-    merged: list[CrawlUrlResult] = []
-    crawler_index = 0
-    for index, _url in enumerate(urls):
-        blocked = blocked_by_index.get(index)
-        if blocked is not None:
-            merged.append(blocked)
-            continue
-
-        merged.append(crawler_results[crawler_index])
-        crawler_index += 1
-
-    return merged