unique-search-proxy 2026.42.0.dev0__tar.gz → 2026.42.0.dev2__tar.gz

{unique_search_proxy-2026.42.0.dev0 → unique_search_proxy-2026.42.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy
-Version: 2026.42.0.dev0
+Version: 2026.42.0.dev2
 Summary: Web Search Proxy implementation
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>
@@ -19,8 +19,8 @@ Requires-Dist: certifi>=2025.11.12,<2027
 Requires-Dist: google-genai>=1.73.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
 Requires-Dist: jinja2>=3.1.6,<4
-Requires-Dist: unique-toolkit[monitoring]>=2026.42.0.dev0,<2026.42.0rc0
-Requires-Dist: unique-search-proxy-core>=2026.42.0.dev0,<2026.42.0rc0
+Requires-Dist: unique-toolkit[monitoring]>=2026.42.0.dev13,<2026.42.0rc0
+Requires-Dist: unique-search-proxy-core>=2026.42.0.dev2,<2026.42.0rc0
 Requires-Dist: openai>=1.105.0,<3
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -237,8 +237,16 @@ Settings use pydantic-settings with per-provider env vars. Copy `.env.example` t
 | VertexAI agent | `VERTEXAI_AGENT_` | `VERTEXAI_AGENT_SERVICE_ACCOUNT_CREDENTIALS` (optional) |
 | HTTP client | `HTTP_CLIENT_` | `HTTP_CLIENT_PROXY_HOST`, `HTTP_CLIENT_POOL_TIMEOUT_SECONDS` |
 | Prometheus | `PROMETHEUS_` | `PROMETHEUS_ENABLED` |
+| URL safety | `URL_SAFETY_` | `URL_SAFETY_ENABLED` |
 | Container | (shell) | `HOST`, `PORT`, `WORKERS`, `LOG_LEVEL` |
 
+`URL_SAFETY_ENABLED` defaults to `true` when unset. Setting it to `false`
+disables application-level SSRF checks and is accepted only when every request
+uses a configured authenticated corporate proxy; incompatible direct or partial
+proxy routing fails during startup. The application cannot verify that the
+proxy implements equivalent URL-safety controls, so this state is highlighted
+in the Prometheus metric and Grafana dashboard.
+
 With `WORKERS > 1`, the entrypoint sets `PROMETHEUS_MULTIPROC_DIR` for correct metric aggregation.
 
 ---

{unique_search_proxy-2026.42.0.dev0 → unique_search_proxy-2026.42.0.dev2}/README.md

@@ -210,8 +210,16 @@ Settings use pydantic-settings with per-provider env vars. Copy `.env.example` t
 | VertexAI agent | `VERTEXAI_AGENT_` | `VERTEXAI_AGENT_SERVICE_ACCOUNT_CREDENTIALS` (optional) |
 | HTTP client | `HTTP_CLIENT_` | `HTTP_CLIENT_PROXY_HOST`, `HTTP_CLIENT_POOL_TIMEOUT_SECONDS` |
 | Prometheus | `PROMETHEUS_` | `PROMETHEUS_ENABLED` |
+| URL safety | `URL_SAFETY_` | `URL_SAFETY_ENABLED` |
 | Container | (shell) | `HOST`, `PORT`, `WORKERS`, `LOG_LEVEL` |
 
+`URL_SAFETY_ENABLED` defaults to `true` when unset. Setting it to `false`
+disables application-level SSRF checks and is accepted only when every request
+uses a configured authenticated corporate proxy; incompatible direct or partial
+proxy routing fails during startup. The application cannot verify that the
+proxy implements equivalent URL-safety controls, so this state is highlighted
+in the Prometheus metric and Grafana dashboard.
+
 With `WORKERS > 1`, the entrypoint sets `PROMETHEUS_MULTIPROC_DIR` for correct metric aggregation.
 
 ---

{unique_search_proxy-2026.42.0.dev0 → unique_search_proxy-2026.42.0.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy"
-version = "2026.42.0.dev0"
+version = "2026.42.0.dev2"
 description = "Web Search Proxy implementation"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"
@@ -21,8 +21,8 @@ dependencies = [
     "google-genai>=1.73.0,<2",
     "google-auth>=2.43.0,<3",
     "jinja2>=3.1.6,<4",
-    "unique-toolkit[monitoring]>=2026.42.0.dev0,<2026.42.0rc0",
-    "unique-search-proxy-core>=2026.42.0.dev0,<2026.42.0rc0",
+    "unique-toolkit[monitoring]>=2026.42.0.dev13,<2026.42.0rc0",
+    "unique-search-proxy-core>=2026.42.0.dev2,<2026.42.0rc0",
     "openai>=1.105.0,<3",
 ]
 

{unique_search_proxy-2026.42.0.dev0 → unique_search_proxy-2026.42.0.dev2}/unique_search_proxy_client/web/core/crawlers/basic/service.py

@@ -15,9 +15,11 @@ from unique_search_proxy_core.schema import (
     ProxyErrorCode,
 )
 from unique_search_proxy_core.url_safety import (
+    CrawlTargetValidationError,
     ResolvedCrawlTarget,
     bypass_crawl_target,
-    pinned_httpx_get_args,
+    safe_pinned_httpx_get,
+    url_safety_settings,
 )
 
 from unique_search_proxy_client.web.core.crawlers.basic.processing import (
@@ -29,10 +31,12 @@ from unique_search_proxy_client.web.core.crawlers.basic.user_agent import (
     random_user_agent,
 )
 from unique_search_proxy_client.web.core.provider_response import (
+    crawl_forbidden_target,
     crawl_upstream_error,
     transport_error_raw,
 )
 from unique_search_proxy_client.web.core.url_safety.gate import AllowedCrawlTarget
+from unique_search_proxy_client.web.monitoring.metrics import record_crawl_blocked
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -101,18 +105,24 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
         semaphore: asyncio.Semaphore,
         content_type_handlers: dict[str, ContentTypeHandlerPolicy],
     ) -> CrawlUrlResult:
-        request_url, pin_headers, extensions = pinned_httpx_get_args(resolved_target)
         async with semaphore:
-            headers = {"User-Agent": random_user_agent(), **pin_headers}
-
             try:
-                response = await client.get(
-                    request_url,
-                    headers=headers,
-                    extensions=extensions or None,
+                response = await safe_pinned_httpx_get(
+                    client,
+                    resolved_target,
+                    headers={"User-Agent": random_user_agent()},
                     timeout=Timeout(timeout),
-                    follow_redirects=True,
+                    max_redirect_hops=url_safety_settings.max_redirect_hops,
+                    enforce_url_safety=url_safety_settings.enabled,
+                )
+            except CrawlTargetValidationError as exc:
+                blocked = exc.blocked_targets[0]
+                record_crawl_blocked(blocked.category)
+                _LOGGER.warning(
+                    "Basic crawl blocked redirect: %s",
+                    blocked.category,
                 )
+                return crawl_forbidden_target(display_url, blocked.reason)
             except httpx.TimeoutException as exc:
                 _LOGGER.warning("Basic crawl timed out for %s: %s", display_url, exc)
                 return crawl_upstream_error(

{unique_search_proxy-2026.42.0.dev0 → unique_search_proxy-2026.42.0.dev2}/unique_search_proxy_client/web/monitoring/metrics.py

@@ -93,6 +93,12 @@ crawl_blocked_total = m.counter(
     "Crawl targets blocked by URL safety policy",
     ["reason_category"],
 )
+url_safety_application_checks_enabled = m.gauge(
+    "url_safety_application_checks_enabled",
+    "Whether application-level URL safety checks are enabled (1) or delegated "
+    "to a configured corporate proxy (0)",
+    [],
+)
 
 agent_search_duration_seconds = m.histogram(
     "agent_search_duration_seconds",
@@ -182,6 +188,12 @@ def record_crawl_blocked(reason_category: str, count: int = 1) -> None:
     crawl_blocked_total.labels(reason_category=reason_category).inc(count)
 
 
+def set_url_safety_application_checks_enabled(enabled: bool) -> None:
+    if not _metrics_enabled():
+        return
+    url_safety_application_checks_enabled.set(int(enabled))
+
+
 def record_agent_search_success(engine: str, duration_seconds: float) -> None:
     if not _metrics_enabled():
         return

{unique_search_proxy-2026.42.0.dev0 → unique_search_proxy-2026.42.0.dev2}/unique_search_proxy_client/web/monitoring/setup.py

@@ -4,9 +4,13 @@ import logging
 
 from fastapi import FastAPI
 from fastapi.responses import PlainTextResponse
+from unique_search_proxy_core.url_safety import url_safety_settings
 
 import unique_search_proxy_client.web.settings.monitoring as monitoring_settings
-from unique_search_proxy_client.web.monitoring.metrics import HTTP_LATENCY_BUCKETS
+from unique_search_proxy_client.web.monitoring.metrics import (
+    HTTP_LATENCY_BUCKETS,
+    set_url_safety_application_checks_enabled,
+)
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -50,5 +54,6 @@ def setup_prometheus(app: FastAPI) -> bool:
             media_type="text/plain; version=0.0.4; charset=utf-8",
         )
 
+    set_url_safety_application_checks_enabled(url_safety_settings.enabled)
     _LOGGER.info("Prometheus metrics enabled at /metrics")
     return True