unique-search-proxy 2026.32.0.dev2__tar.gz → 2026.32.0.dev3__tar.gz

{unique_search_proxy-2026.32.0.dev2 → unique_search_proxy-2026.32.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy
-Version: 2026.32.0.dev2
+Version: 2026.32.0.dev3
 Summary: Web Search Proxy implementation
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>
@@ -19,8 +19,8 @@ Requires-Dist: certifi>=2025.11.12,<2027
 Requires-Dist: google-genai>=1.73.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
 Requires-Dist: jinja2>=3.1.6,<4
-Requires-Dist: unique-toolkit[monitoring]>=2026.32.0.dev2,<2026.32.0rc0
-Requires-Dist: unique-search-proxy-core>=2026.32.0.dev0,<2026.32.0rc0
+Requires-Dist: unique-toolkit[monitoring]>=2026.32.0.dev3,<2026.32.0rc0
+Requires-Dist: unique-search-proxy-core>=2026.32.0.dev1,<2026.32.0rc0
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 

{unique_search_proxy-2026.32.0.dev2 → unique_search_proxy-2026.32.0.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy"
-version = "2026.32.0.dev2"
+version = "2026.32.0.dev3"
 description = "Web Search Proxy implementation"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"
@@ -21,8 +21,8 @@ dependencies = [
     "google-genai>=1.73.0,<2",
     "google-auth>=2.43.0,<3",
     "jinja2>=3.1.6,<4",
-    "unique-toolkit[monitoring]>=2026.32.0.dev2,<2026.32.0rc0",
-    "unique-search-proxy-core>=2026.32.0.dev0,<2026.32.0rc0",
+    "unique-toolkit[monitoring]>=2026.32.0.dev3,<2026.32.0rc0",
+    "unique-search-proxy-core>=2026.32.0.dev1,<2026.32.0rc0",
 ]
 
 [dependency-groups]

{unique_search_proxy-2026.32.0.dev2 → unique_search_proxy-2026.32.0.dev3}/unique_search_proxy_client/web/api/v1/crawl.py

@@ -11,7 +11,11 @@ from unique_search_proxy_core.errors import (
     UpstreamTimeoutError,
     attach_request_context,
 )
-from unique_search_proxy_core.schema import CrawlResponse, ProxyErrorCode
+from unique_search_proxy_core.schema import (
+    CrawlResponse,
+    CrawlUrlResult,
+    ProxyErrorCode,
+)
 
 from unique_search_proxy_client.web.api.v1.openapi_examples import (
     CRAWL_OPENAPI_EXAMPLES,
@@ -28,6 +32,7 @@ from unique_search_proxy_client.web.core.url_safety.gate import (
 from unique_search_proxy_client.web.monitoring.metrics import (
     record_crawl_error,
     record_crawl_success,
+    record_crawl_url_outcomes,
 )
 
 router = APIRouter(tags=["crawl"])
@@ -42,6 +47,22 @@ def _crawl_request_context(exc: ProxyError, *, crawler_id: str) -> ProxyError:
     )
 
 
+def _url_outcomes(results: list[CrawlUrlResult]) -> list[tuple[str, str, str]]:
+    """Derive ``(outcome, error_code, http_status)`` triples from merged results."""
+    outcomes: list[tuple[str, str, str]] = []
+    for result in results:
+        if result.error is None:
+            outcomes.append(("success", "", ""))
+            continue
+        http_status = (
+            str(result.error.status_code)
+            if result.error.status_code is not None
+            else ""
+        )
+        outcomes.append(("error", result.error.code, http_status))
+    return outcomes
+
+
 @router.post(
     "/crawl",
     response_model=CrawlResponse,
@@ -71,6 +92,15 @@ async def crawl(
                     len(body.urls),
                     duration,
                 )
+                merged_results = merge_crawl_results(
+                    body.urls,
+                    blocked_by_index=gate.blocked_by_index,
+                    crawler_results=[],
+                )
+                record_crawl_url_outcomes(
+                    crawler_id,
+                    _url_outcomes(merged_results),
+                )
                 _LOGGER.info(
                     "crawl success crawler=%s urls=%d blocked=%d results=0 duration=%.0fms",
                     crawler_id,
@@ -80,11 +110,7 @@ async def crawl(
                 )
                 return CrawlResponse(
                     crawler=crawler_id,
-                    results=merge_crawl_results(
-                        body.urls,
-                        blocked_by_index=gate.blocked_by_index,
-                        crawler_results=[],
-                    ),
+                    results=merged_results,
                 )
 
             crawl_body = body.model_copy(
@@ -121,6 +147,11 @@ async def crawl(
             crawler_id=crawler_id,
         ) from exc
     except ProxyError as exc:
+        record_crawl_error(
+            crawler_id,
+            exc.code.value if hasattr(exc.code, "value") else str(exc.code),
+            time.perf_counter() - started,
+        )
         _LOGGER.warning(
             "crawl failed crawler=%s code=%s duration=%.0fms",
             crawler_id,
@@ -148,6 +179,7 @@ async def crawl(
         blocked_by_index=gate.blocked_by_index,
         crawler_results=crawler_results,
     )
+    record_crawl_url_outcomes(crawler_id, _url_outcomes(merged_results))
     _LOGGER.info(
         "crawl success crawler=%s urls=%d blocked=%d results=%d duration=%.0fms",
         crawler_id,

{unique_search_proxy-2026.32.0.dev2 → unique_search_proxy-2026.32.0.dev3}/unique_search_proxy_client/web/api/v1/search.py

@@ -93,6 +93,11 @@ async def search(
             engine_id=engine_id,
         ) from exc
     except ProxyError as exc:
+        record_search_error(
+            engine_id,
+            exc.code.value if hasattr(exc.code, "value") else str(exc.code),
+            time.perf_counter() - started,
+        )
         _LOGGER.warning(
             "search failed engine=%s code=%s duration=%.0fms",
             engine_id,

{unique_search_proxy-2026.32.0.dev2 → unique_search_proxy-2026.32.0.dev3}/unique_search_proxy_client/web/core/crawlers/basic/service.py

@@ -144,6 +144,7 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
                     f"HTTP {response.status_code} while fetching URL",
                     content_type=content_type,
                     raw=raw_body,
+                    status_code=response.status_code,
                 )
 
             content = await self._maybe_process_content(

{unique_search_proxy-2026.32.0.dev2 → unique_search_proxy-2026.32.0.dev3}/unique_search_proxy_client/web/core/provider_response.py

@@ -69,13 +69,14 @@ def crawl_upstream_error(
     raw: Any | None = None,
     content_type: str | None = "text/markdown",
     code: str = ProxyErrorCode.UPSTREAM_ERROR.value,
+    status_code: int | None = None,
 ) -> CrawlUrlResult:
     """Build a per-URL crawl failure with optional upstream response attached."""
     return CrawlUrlResult(
         url=url,
         content=None,
         content_type=content_type,
-        error=PerUrlError(code=code, message=message),
+        error=PerUrlError(code=code, message=message, status_code=status_code),
         raw=raw,
     )
 

{unique_search_proxy-2026.32.0.dev2 → unique_search_proxy-2026.32.0.dev3}/unique_search_proxy_client/web/error_handlers.py

@@ -37,16 +37,10 @@ def register_exception_handlers(app: FastAPI) -> None:
     @app.exception_handler(ProxyError)
     async def proxy_error_handler(_request: Request, exc: ProxyError) -> JSONResponse:
         from unique_search_proxy_client.web.monitoring.metrics import (
-            record_crawl_error,
             record_proxy_error,
-            record_search_error,
         )
 
         record_proxy_error(exc.code.value)
-        if exc.request == "search" and exc.provider is not None:
-            record_search_error(exc.provider, exc.code.value, 0.0)
-        if exc.request == "crawl" and exc.provider is not None:
-            record_crawl_error(exc.provider, exc.code.value, 0.0)
         _LOGGER.warning(
             "Proxy error [%s] request=%s provider=%s: %s",
             exc.code.value,

{unique_search_proxy-2026.32.0.dev2 → unique_search_proxy-2026.32.0.dev3}/unique_search_proxy_client/web/monitoring/metrics.py

@@ -1,13 +1,35 @@
 from __future__ import annotations
 
+from collections.abc import Iterable
+
 from unique_toolkit.monitoring import MetricNamespace
 
 m = MetricNamespace("unique_search_proxy")
 
+
+def _linear_buckets(ceiling: float, steps: int) -> tuple[float, ...]:
+    """Constant-step latency buckets (seconds) up to ``ceiling`` in ``steps``.
+
+    The range ``(0, ceiling]`` is split into ``steps`` equal buckets, yielding
+    ``ceiling/steps, 2*ceiling/steps, ..., ceiling`` inclusive. Observations above
+    the ceiling land in the implicit +Inf bucket prometheus_client appends;
+    histogram_quantile cannot resolve past the largest finite bucket, so each
+    ceiling exceeds the workload's typical timeout. Using the same ``steps`` count
+    keeps resolution comparable across workloads while the ceiling adapts to range.
+    """
+    step = ceiling / steps
+    return tuple(round(step * i, 3) for i in range(1, steps + 1))
+
+
+_SEARCH_LATENCY_BUCKETS = _linear_buckets(ceiling=5.0, steps=20)
+_CRAWL_LATENCY_BUCKETS = _linear_buckets(ceiling=60.0, steps=20)
+_AGENT_SEARCH_LATENCY_BUCKETS = _linear_buckets(ceiling=120.0, steps=20)
+
 search_duration_seconds = m.histogram(
     "search_duration_seconds",
     "Search request latency",
     ["engine"],
+    buckets=_SEARCH_LATENCY_BUCKETS,
 )
 search_total = m.counter(
     "search_total",
@@ -24,6 +46,7 @@ crawl_duration_seconds = m.histogram(
     "crawl_duration_seconds",
     "Crawl request latency",
     ["crawler"],
+    buckets=_CRAWL_LATENCY_BUCKETS,
 )
 crawl_total = m.counter(
     "crawl_total",
@@ -40,6 +63,11 @@ crawl_urls_total = m.counter(
     "URLs submitted to crawl",
     ["crawler"],
 )
+crawl_url_outcomes_total = m.counter(
+    "crawl_url_outcomes_total",
+    "Per-URL crawl outcomes",
+    ["crawler", "outcome", "error_code", "http_status"],
+)
 
 crawl_blocked_total = m.counter(
     "crawl_blocked_total",
@@ -51,6 +79,7 @@ agent_search_duration_seconds = m.histogram(
     "agent_search_duration_seconds",
     "Agent search request latency",
     ["engine"],
+    buckets=_AGENT_SEARCH_LATENCY_BUCKETS,
 )
 agent_search_total = m.counter(
     "agent_search_total",
@@ -105,6 +134,29 @@ def record_crawl_error(crawler: str, error_code: str, duration_seconds: float) -
     crawl_duration_seconds.labels(crawler=crawler).observe(duration_seconds)
 
 
+def record_crawl_url_outcomes(
+    crawler: str,
+    outcomes: Iterable[tuple[str, str, str]],
+) -> None:
+    """Record per-URL crawl outcomes.
+
+    Each item is an ``(outcome, error_code, http_status)`` triple where
+    ``outcome`` is ``"success"`` or ``"error"``, ``error_code`` is ``""`` on
+    success or the per-URL :class:`PerUrlError` code otherwise, and
+    ``http_status`` is the upstream HTTP status (e.g. ``"403"``) when the failure
+    was an HTTP error or ``""`` when no HTTP response was received.
+    """
+    if not _metrics_enabled():
+        return
+    for outcome, error_code, http_status in outcomes:
+        crawl_url_outcomes_total.labels(
+            crawler=crawler,
+            outcome=outcome,
+            error_code=error_code,
+            http_status=http_status,
+        ).inc()
+
+
 def record_crawl_blocked(reason_category: str, count: int = 1) -> None:
     if not _metrics_enabled():
         return