unique-search-proxy 2026.46.0.dev5__tar.gz → 2026.46.0.dev7__tar.gz

{unique_search_proxy-2026.46.0.dev5 → unique_search_proxy-2026.46.0.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy
-Version: 2026.46.0.dev5
+Version: 2026.46.0.dev7
 Summary: Web Search Proxy implementation
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>
@@ -20,8 +20,8 @@ Requires-Dist: certifi>=2025.11.12,<2027
 Requires-Dist: google-genai>=1.73.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
 Requires-Dist: jinja2>=3.1.6,<4
-Requires-Dist: unique-toolkit[monitoring]>=2026.46.0.dev4,<2026.46.0rc0
-Requires-Dist: unique-search-proxy-core>=2026.46.0.dev1,<2026.46.0rc0
+Requires-Dist: unique-toolkit[monitoring]>=2026.46.0.dev10,<2026.46.0rc0
+Requires-Dist: unique-search-proxy-core>=2026.46.0.dev3,<2026.46.0rc0
 Requires-Dist: openai>=1.105.0,<3
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown

{unique_search_proxy-2026.46.0.dev5 → unique_search_proxy-2026.46.0.dev7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy"
-version = "2026.46.0.dev5"
+version = "2026.46.0.dev7"
 description = "Web Search Proxy implementation"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"
@@ -22,8 +22,8 @@ dependencies = [
     "google-genai>=1.73.0,<2",
     "google-auth>=2.43.0,<3",
     "jinja2>=3.1.6,<4",
-    "unique-toolkit[monitoring]>=2026.46.0.dev4,<2026.46.0rc0",
-    "unique-search-proxy-core>=2026.46.0.dev1,<2026.46.0rc0",
+    "unique-toolkit[monitoring]>=2026.46.0.dev10,<2026.46.0rc0",
+    "unique-search-proxy-core>=2026.46.0.dev3,<2026.46.0rc0",
     "openai>=1.105.0,<3",
 ]
 

{unique_search_proxy-2026.46.0.dev5 → unique_search_proxy-2026.46.0.dev7}/unique_search_proxy_client/web/core/url_safety/gate.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import time
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
@@ -16,7 +17,13 @@ from unique_search_proxy_client.web.core.provider_response import (
     crawl_deadline_missed,
     crawl_forbidden_target,
 )
-from unique_search_proxy_client.web.monitoring.metrics import record_crawl_blocked
+from unique_search_proxy_client.web.monitoring.metrics import (
+    record_crawl_blocked,
+    record_url_safety_check,
+)
+from unique_search_proxy_client.web.settings.request_limits import (
+    request_limits_settings,
+)
 
 if TYPE_CHECKING:
     from httpx import AsyncClient
@@ -50,32 +57,30 @@ async def apply_url_safety_gate(
     deadline: float | None = None,
 ) -> UrlSafetyGateResult:
     """Partition crawl URLs into allowed, blocked and timed-out targets."""
+    check_slots = asyncio.Semaphore(request_limits_settings.max_concurrent_url_checks)
+    outcomes = await asyncio.gather(
+        *[
+            _validate_before_deadline(
+                url,
+                redirect_http_client=redirect_http_client,
+                check_slots=check_slots,
+                deadline=deadline,
+            )
+            for url in urls
+        ],
+    )
+
     allowed_targets: list[AllowedCrawlTarget] = []
     blocked_by_index: dict[int, CrawlUrlResult] = {}
     timed_out_by_index: dict[int, CrawlUrlResult] = {}
 
-    for index, url in enumerate(urls):
-        outcome = await _validate_before_deadline(
-            url,
-            redirect_http_client=redirect_http_client,
-            deadline=deadline,
-        )
+    for index, (url, outcome) in enumerate(zip(urls, outcomes, strict=True)):
         if outcome is None:
-            timed_out_by_index = {
-                unchecked_index: crawl_deadline_missed(
-                    unchecked_url.strip(),
-                    "URL safety check did not finish before the crawl deadline",
-                )
-                for unchecked_index, unchecked_url in enumerate(
-                    urls[index:],
-                    start=index,
-                )
-            }
-            _LOGGER.warning(
-                "URL safety checks missed the crawl deadline unchecked=%d",
-                len(timed_out_by_index),
+            timed_out_by_index[index] = crawl_deadline_missed(
+                url.strip(),
+                "URL safety check did not finish before the crawl deadline",
             )
-            break
+            continue
 
         if outcome.blocked is not None:
             record_crawl_blocked(outcome.blocked.category)
@@ -96,6 +101,12 @@ async def apply_url_safety_gate(
             ),
         )
 
+    if timed_out_by_index:
+        _LOGGER.warning(
+            "URL safety checks missed the crawl deadline unchecked=%d",
+            len(timed_out_by_index),
+        )
+
     return UrlSafetyGateResult(
         allowed_targets=allowed_targets,
         blocked_by_index=blocked_by_index,
@@ -107,18 +118,37 @@ async def _validate_before_deadline(
     url: str,
     *,
     redirect_http_client: AsyncClient,
+    check_slots: asyncio.Semaphore,
     deadline: float | None,
 ) -> UrlSafetyOutcome | None:
     """Return the URL's safety outcome, or ``None`` when the deadline passes first."""
     try:
-        async with asyncio.timeout_at(deadline):
-            [outcome] = await UrlSafetyService.validate_urls_individually(
-                [url],
+        async with asyncio.timeout_at(deadline), check_slots:
+            return await _validate_and_record(
+                url,
                 redirect_http_client=redirect_http_client,
             )
     except TimeoutError:
         return None
-    return outcome
+
+
+async def _validate_and_record(
+    url: str,
+    *,
+    redirect_http_client: AsyncClient,
+) -> UrlSafetyOutcome:
+    """Validate one URL and record how long the check took, by outcome."""
+    started = time.perf_counter()
+    outcome_label = "unfinished"
+    try:
+        [outcome] = await UrlSafetyService.validate_urls_individually(
+            [url],
+            redirect_http_client=redirect_http_client,
+        )
+        outcome_label = "allowed" if outcome.blocked is None else "blocked"
+        return outcome
+    finally:
+        record_url_safety_check(outcome_label, time.perf_counter() - started)
 
 
 def merge_crawl_results(

{unique_search_proxy-2026.46.0.dev5 → unique_search_proxy-2026.46.0.dev7}/unique_search_proxy_client/web/monitoring/metrics.py

@@ -40,6 +40,9 @@ def _exponential_buckets(start: float, ceiling: float, count: int) -> tuple[floa
 _SEARCH_LATENCY_BUCKETS = _linear_buckets(ceiling=5.0, steps=20)
 _CRAWL_LATENCY_BUCKETS = _linear_buckets(ceiling=60.0, steps=20)
 _AGENT_SEARCH_LATENCY_BUCKETS = _linear_buckets(ceiling=120.0, steps=20)
+# Half-second steps up to 10s, so checks that hit the per-hop redirect timeout
+# (5s by default) show up as their own bucket.
+_URL_SAFETY_CHECK_LATENCY_BUCKETS = _linear_buckets(ceiling=10.0, steps=20)
 
 # One histogram covers every endpoint, from sub-second /v1/search to /v1/agent-search
 # runs approaching 120s — exponential buckets keep resolution useful at both ends.
@@ -94,6 +97,12 @@ crawl_blocked_total = m.counter(
     "Crawl targets blocked by URL safety policy",
     ["reason_category"],
 )
+url_safety_check_duration_seconds = m.histogram(
+    "url_safety_check_duration_seconds",
+    "Time to check one crawl URL (redirect probes and DNS), by outcome",
+    ["outcome"],
+    buckets=_URL_SAFETY_CHECK_LATENCY_BUCKETS,
+)
 url_safety_application_checks_enabled = m.gauge(
     "url_safety_application_checks_enabled",
     "Whether application-level URL safety checks are enabled (1) or delegated "
@@ -182,6 +191,11 @@ def record_crawl_blocked(reason_category: str, count: int = 1) -> None:
     crawl_blocked_total.labels(reason_category=reason_category).inc(count)
 
 
+def record_url_safety_check(outcome: str, duration_seconds: float) -> None:
+    """Record one URL check; ``outcome`` is allowed, blocked or unfinished."""
+    url_safety_check_duration_seconds.labels(outcome=outcome).observe(duration_seconds)
+
+
 def set_url_safety_application_checks_enabled(enabled: bool) -> None:
     url_safety_application_checks_enabled.set(int(enabled))
 

{unique_search_proxy-2026.46.0.dev5 → unique_search_proxy-2026.46.0.dev7}/unique_search_proxy_client/web/settings/request_limits.py

@@ -23,6 +23,7 @@ REQUEST_LIMITS_ENV_PREFIX = "REQUEST_LIMITS_"
             "max_url_length",
             "max_concurrent_fetches",
             "max_concurrent_crawls",
+            "max_concurrent_url_checks",
         ],
         "timeouts": ["max_timeout_seconds"],
     },
@@ -82,6 +83,14 @@ class RequestLimitsSettings(BaseSettings):
             "this get a 429 instead of queueing."
         ),
     )
+    max_concurrent_url_checks: int = Field(
+        default=3,
+        ge=1,
+        description=(
+            "URL safety checks one crawl request runs at once before fetching. "
+            "Each check probes the URL's redirects with HEAD requests."
+        ),
+    )
     max_timeout_seconds: int = Field(
         default=120,
         ge=1,