unique-search-proxy 2026.26.0.dev9__tar.gz → 2026.26.0.dev10__tar.gz

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy
-Version: 2026.26.0.dev9
+Version: 2026.26.0.dev10
 Summary: Web Search Proxy implementation
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>
@@ -19,7 +19,7 @@ Requires-Dist: certifi>=2025.11.12,<2027
 Requires-Dist: google-genai>=1.73.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
 Requires-Dist: unique-toolkit[monitoring]>=2026.26.0.dev11,<2026.26.0rc0
-Requires-Dist: unique-search-proxy-core>=2026.26.0.dev5,<2026.26.0rc0
+Requires-Dist: unique-search-proxy-core>=2026.26.0.dev6,<2026.26.0rc0
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy"
-version = "2026.26.0.dev9"
+version = "2026.26.0.dev10"
 description = "Web Search Proxy implementation"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"
@@ -21,7 +21,7 @@ dependencies = [
     "google-genai>=1.73.0,<2",
     "google-auth>=2.43.0,<3",
     "unique-toolkit[monitoring]>=2026.26.0.dev11,<2026.26.0rc0",
-    "unique-search-proxy-core>=2026.26.0.dev5,<2026.26.0rc0",
+    "unique-search-proxy-core>=2026.26.0.dev6,<2026.26.0rc0",
 ]
 
 [dependency-groups]

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev10}/unique_search_proxy_client/web/api/v1/crawl.py

@@ -18,6 +18,13 @@ from unique_search_proxy_client.web.api.v1.openapi_examples import (
 )
 from unique_search_proxy_client.web.core.client import get_http_client_pool
 from unique_search_proxy_client.web.core.crawlers.factory import get_crawler_service
+from unique_search_proxy_client.web.core.crawlers.pinned_egress import (
+    PinnedEgressCrawler,
+)
+from unique_search_proxy_client.web.core.url_safety.gate import (
+    apply_url_safety_gate,
+    merge_crawl_results,
+)
 from unique_search_proxy_client.web.monitoring.metrics import (
     record_crawl_error,
     record_crawl_success,
@@ -49,10 +56,38 @@ async def crawl(
     started = time.perf_counter()
 
     try:
-        pool = get_http_client_pool(request.app)
-        crawler = get_crawler_service(crawler_id, http_client=pool.client)
         async with asyncio.timeout(timeout):
-            results = await crawler.crawl(body)
+            gate = await apply_url_safety_gate(body.urls)
+            if not gate.allowed_targets:
+                record_crawl_success(
+                    crawler_id,
+                    len(body.urls),
+                    time.perf_counter() - started,
+                )
+                return CrawlResponse(
+                    crawler=crawler_id,
+                    results=merge_crawl_results(
+                        body.urls,
+                        blocked_by_index=gate.blocked_by_index,
+                        crawler_results=[],
+                    ),
+                )
+
+            crawl_body = body.model_copy(
+                update={
+                    "urls": [target.display_url for target in gate.allowed_targets],
+                },
+            )
+
+            pool = get_http_client_pool(request.app)
+            crawler = get_crawler_service(crawler_id, http_client=pool.client)
+            if isinstance(crawler, PinnedEgressCrawler):
+                crawler_results = await crawler.crawl_pinned(
+                    crawl_body,
+                    gate.allowed_targets,
+                )
+            else:
+                crawler_results = await crawler.crawl(crawl_body)
     except TimeoutError as exc:
         record_crawl_error(
             crawler_id,
@@ -76,4 +111,11 @@ async def crawl(
         raise
 
     record_crawl_success(crawler_id, len(body.urls), time.perf_counter() - started)
-    return CrawlResponse(crawler=crawler_id, results=results)
+    return CrawlResponse(
+        crawler=crawler_id,
+        results=merge_crawl_results(
+            body.urls,
+            blocked_by_index=gate.blocked_by_index,
+            crawler_results=crawler_results,
+        ),
+    )

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev10}/unique_search_proxy_client/web/core/crawlers/basic/service.py

@@ -14,6 +14,11 @@ from unique_search_proxy_core.schema import (
     CrawlUrlResult,
     ProxyErrorCode,
 )
+from unique_search_proxy_core.url_safety import (
+    ResolvedCrawlTarget,
+    bypass_crawl_target,
+    pinned_httpx_get_args,
+)
 
 from unique_search_proxy_client.web.core.crawlers.basic.processing import (
     ContentProcessingError,
@@ -27,6 +32,7 @@ from unique_search_proxy_client.web.core.provider_response import (
     crawl_upstream_error,
     transport_error_raw,
 )
+from unique_search_proxy_client.web.core.url_safety.gate import AllowedCrawlTarget
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -43,11 +49,30 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
 
     crawler_id = CrawlerType.BASIC.value
 
-    async def crawl(self, request: BasicCrawlRequest) -> list[CrawlUrlResult]:  # type: ignore
+    async def crawl(self, request: BasicCrawlRequest) -> list[CrawlUrlResult]:  # type: ignore[override]
+        bypass_targets = [
+            AllowedCrawlTarget(
+                display_url=url.strip(),
+                resolved=bypass_crawl_target(url),
+            )
+            for url in request.urls
+        ]
+        return await self.crawl_pinned(request, bypass_targets)
+
+    async def crawl_pinned(
+        self,
+        request: BasicCrawlRequest,  # type: ignore[valid-type]
+        allowed_targets: list[AllowedCrawlTarget],
+    ) -> list[CrawlUrlResult]:
         client = self._http_client
         if client is None:
             raise RuntimeError("HTTP client is required for Basic crawler")
 
+        display_urls = list(request.urls)
+        if len(allowed_targets) != len(display_urls):
+            msg = "allowed_targets length must match request.urls length"
+            raise ValueError(msg)
+
         timeout = request.timeout
         semaphore = asyncio.Semaphore(request.max_concurrent_requests)
         return list(
@@ -55,12 +80,13 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
                 *[
                     self._crawl_one(
                         client,
-                        url,
+                        allowed_target.display_url,
+                        resolved_target=allowed_target.resolved,
                         timeout=timeout,
                         semaphore=semaphore,
                         content_type_handlers=request.content_types.to_handlers(),
                     )
-                    for url in request.urls
+                    for allowed_target in allowed_targets
                 ],
             ),
         )
@@ -68,35 +94,38 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
     async def _crawl_one(
         self,
         client: AsyncClient,
-        url: str,
+        display_url: str,
         *,
+        resolved_target: ResolvedCrawlTarget,
         timeout: int,
         semaphore: asyncio.Semaphore,
         content_type_handlers: dict[str, ContentTypeHandlerPolicy],
     ) -> CrawlUrlResult:
-        request_url = url.strip()
+        request_url, pin_headers, extensions = pinned_httpx_get_args(resolved_target)
         async with semaphore:
-            headers = {"User-Agent": random_user_agent()}
+            headers = {"User-Agent": random_user_agent(), **pin_headers}
+
             try:
                 response = await client.get(
                     request_url,
                     headers=headers,
+                    extensions=extensions or None,
                     timeout=Timeout(timeout),
                     follow_redirects=True,
                 )
             except httpx.TimeoutException as exc:
-                _LOGGER.warning("Basic crawl timed out for %s: %s", request_url, exc)
+                _LOGGER.warning("Basic crawl timed out for %s: %s", display_url, exc)
                 return crawl_upstream_error(
-                    request_url,
+                    display_url,
                     f"Crawl timed out after {timeout}s",
                     content_type=None,
                     code=ProxyErrorCode.UPSTREAM_TIMEOUT.value,
                     raw=transport_error_raw(exc),
                 )
             except httpx.HTTPError as exc:
-                _LOGGER.warning("Basic crawl failed for %s: %s", request_url, exc)
+                _LOGGER.warning("Basic crawl failed for %s: %s", display_url, exc)
                 return crawl_upstream_error(
-                    request_url,
+                    display_url,
                     str(exc),
                     content_type=None,
                     raw=transport_error_raw(exc),
@@ -107,11 +136,11 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
             if response.is_error:
                 _LOGGER.warning(
                     "Basic crawl HTTP error for %s: %s",
-                    request_url,
+                    display_url,
                     response.status_code,
                 )
                 return crawl_upstream_error(
-                    request_url,
+                    display_url,
                     f"HTTP {response.status_code} while fetching URL",
                     content_type=content_type,
                     raw=raw_body,
@@ -120,7 +149,7 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
             content = await self._maybe_process_content(
                 raw_body,
                 content_type,
-                request_url=request_url,
+                request_url=display_url,
                 timeout=timeout,
                 content_type_handlers=content_type_handlers,
             )
@@ -128,7 +157,7 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
                 return content
 
             return CrawlUrlResult(
-                url=request_url,
+                url=display_url,
                 content=content,
                 raw=raw_body,
                 content_type=content_type,

unique_search_proxy-2026.26.0.dev10/unique_search_proxy_client/web/core/crawlers/pinned_egress.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+from pydantic import BaseModel
+from unique_search_proxy_core.schema import CrawlUrlResult
+
+from unique_search_proxy_client.web.core.url_safety.gate import AllowedCrawlTarget
+
+
+@runtime_checkable
+class PinnedEgressCrawler(Protocol):
+    """Crawlers that fetch directly and must reuse gate DNS resolution."""
+
+    async def crawl_pinned(
+        self,
+        request: BaseModel,
+        allowed_targets: list[AllowedCrawlTarget],
+    ) -> list[CrawlUrlResult]: ...
+
+
+__all__ = ["PinnedEgressCrawler"]

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev10}/unique_search_proxy_client/web/core/provider_response.py

@@ -80,6 +80,16 @@ def crawl_upstream_error(
     )
 
 
+def crawl_forbidden_target(url: str, message: str) -> CrawlUrlResult:
+    """Build a per-URL crawl failure for a URL safety policy violation."""
+    return crawl_upstream_error(
+        url,
+        message,
+        code=ProxyErrorCode.FORBIDDEN_TARGET.value,
+        content_type=None,
+    )
+
+
 def raise_for_upstream_response(
     response: httpx.Response,
     *,

unique_search_proxy-2026.26.0.dev10/unique_search_proxy_client/web/core/url_safety/__init__.py

@@ -0,0 +1,13 @@
+from unique_search_proxy_client.web.core.url_safety.gate import (
+    AllowedCrawlTarget,
+    UrlSafetyGateResult,
+    apply_url_safety_gate,
+    merge_crawl_results,
+)
+
+__all__ = [
+    "AllowedCrawlTarget",
+    "UrlSafetyGateResult",
+    "apply_url_safety_gate",
+    "merge_crawl_results",
+]

unique_search_proxy-2026.26.0.dev10/unique_search_proxy_client/web/core/url_safety/gate.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from unique_search_proxy_core.schema import CrawlUrlResult
+from unique_search_proxy_core.url_safety import ResolvedCrawlTarget, UrlSafetyService
+
+from unique_search_proxy_client.web.core.provider_response import crawl_forbidden_target
+from unique_search_proxy_client.web.monitoring.metrics import record_crawl_blocked
+
+
+@dataclass(frozen=True)
+class AllowedCrawlTarget:
+    """User-facing URL paired with the validated resolution for pinned egress."""
+
+    display_url: str
+    resolved: ResolvedCrawlTarget
+
+
+@dataclass(frozen=True)
+class UrlSafetyGateResult:
+    allowed_targets: list[AllowedCrawlTarget]
+    blocked_by_index: dict[int, CrawlUrlResult]
+
+
+async def apply_url_safety_gate(urls: list[str]) -> UrlSafetyGateResult:
+    """Validate crawl URLs and partition them into allowed vs blocked targets."""
+    outcomes = await UrlSafetyService.validate_urls_individually(urls)
+    allowed_targets: list[AllowedCrawlTarget] = []
+    blocked_by_index: dict[int, CrawlUrlResult] = {}
+
+    for index, outcome in enumerate(outcomes):
+        if outcome.blocked is not None:
+            record_crawl_blocked(outcome.blocked.category)
+            blocked_by_index[index] = crawl_forbidden_target(
+                outcome.url.strip(),
+                outcome.blocked.reason,
+            )
+            continue
+
+        if outcome.resolved is None:
+            msg = "URL safety allowed a crawl target without resolved metadata"
+            raise RuntimeError(msg)
+
+        allowed_targets.append(
+            AllowedCrawlTarget(
+                display_url=outcome.url.strip(),
+                resolved=outcome.resolved,
+            ),
+        )
+
+    return UrlSafetyGateResult(
+        allowed_targets=allowed_targets,
+        blocked_by_index=blocked_by_index,
+    )
+
+
+def merge_crawl_results(
+    urls: list[str],
+    *,
+    blocked_by_index: dict[int, CrawlUrlResult],
+    crawler_results: list[CrawlUrlResult],
+) -> list[CrawlUrlResult]:
+    """Merge per-URL blocked results with crawler outcomes in request order."""
+    merged: list[CrawlUrlResult] = []
+    crawler_index = 0
+    for index, _url in enumerate(urls):
+        blocked = blocked_by_index.get(index)
+        if blocked is not None:
+            merged.append(blocked)
+            continue
+
+        merged.append(crawler_results[crawler_index])
+        crawler_index += 1
+
+    return merged