unique-search-proxy 2026.26.0.dev9__tar.gz → 2026.26.0.dev11__tar.gz

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy
-Version: 2026.26.0.dev9
+Version: 2026.26.0.dev11
 Summary: Web Search Proxy implementation
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>
@@ -19,7 +19,7 @@ Requires-Dist: certifi>=2025.11.12,<2027
 Requires-Dist: google-genai>=1.73.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
 Requires-Dist: unique-toolkit[monitoring]>=2026.26.0.dev11,<2026.26.0rc0
-Requires-Dist: unique-search-proxy-core>=2026.26.0.dev5,<2026.26.0rc0
+Requires-Dist: unique-search-proxy-core>=2026.26.0.dev7,<2026.26.0rc0
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy"
-version = "2026.26.0.dev9"
+version = "2026.26.0.dev11"
 description = "Web Search Proxy implementation"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"
@@ -21,7 +21,7 @@ dependencies = [
     "google-genai>=1.73.0,<2",
     "google-auth>=2.43.0,<3",
     "unique-toolkit[monitoring]>=2026.26.0.dev11,<2026.26.0rc0",
-    "unique-search-proxy-core>=2026.26.0.dev5,<2026.26.0rc0",
+    "unique-search-proxy-core>=2026.26.0.dev7,<2026.26.0rc0",
 ]
 
 [dependency-groups]

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/unique_search_proxy_client/web/api/v1/crawl.py

@@ -18,6 +18,13 @@ from unique_search_proxy_client.web.api.v1.openapi_examples import (
 )
 from unique_search_proxy_client.web.core.client import get_http_client_pool
 from unique_search_proxy_client.web.core.crawlers.factory import get_crawler_service
+from unique_search_proxy_client.web.core.crawlers.pinned_egress import (
+    PinnedEgressCrawler,
+)
+from unique_search_proxy_client.web.core.url_safety.gate import (
+    apply_url_safety_gate,
+    merge_crawl_results,
+)
 from unique_search_proxy_client.web.monitoring.metrics import (
     record_crawl_error,
     record_crawl_success,
@@ -49,10 +56,38 @@ async def crawl(
     started = time.perf_counter()
 
     try:
-        pool = get_http_client_pool(request.app)
-        crawler = get_crawler_service(crawler_id, http_client=pool.client)
         async with asyncio.timeout(timeout):
-            results = await crawler.crawl(body)
+            gate = await apply_url_safety_gate(body.urls)
+            if not gate.allowed_targets:
+                record_crawl_success(
+                    crawler_id,
+                    len(body.urls),
+                    time.perf_counter() - started,
+                )
+                return CrawlResponse(
+                    crawler=crawler_id,
+                    results=merge_crawl_results(
+                        body.urls,
+                        blocked_by_index=gate.blocked_by_index,
+                        crawler_results=[],
+                    ),
+                )
+
+            crawl_body = body.model_copy(
+                update={
+                    "urls": [target.display_url for target in gate.allowed_targets],
+                },
+            )
+
+            pool = get_http_client_pool(request.app)
+            crawler = get_crawler_service(crawler_id, http_client=pool.client)
+            if isinstance(crawler, PinnedEgressCrawler):
+                crawler_results = await crawler.crawl_pinned(
+                    crawl_body,
+                    gate.allowed_targets,
+                )
+            else:
+                crawler_results = await crawler.crawl(crawl_body)
     except TimeoutError as exc:
         record_crawl_error(
             crawler_id,
@@ -76,4 +111,11 @@ async def crawl(
         raise
 
     record_crawl_success(crawler_id, len(body.urls), time.perf_counter() - started)
-    return CrawlResponse(crawler=crawler_id, results=results)
+    return CrawlResponse(
+        crawler=crawler_id,
+        results=merge_crawl_results(
+            body.urls,
+            blocked_by_index=gate.blocked_by_index,
+            crawler_results=crawler_results,
+        ),
+    )

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/unique_search_proxy_client/web/app.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import logging
-import os
+import sys
 from contextlib import asynccontextmanager
 
 from dotenv import load_dotenv
@@ -13,22 +13,19 @@ from unique_search_proxy_client.web.api import health_router, v1_router
 from unique_search_proxy_client.web.core.client.service import create_http_client_pool
 from unique_search_proxy_client.web.core.providers import register_builtin_providers
 from unique_search_proxy_client.web.error_handlers import register_exception_handlers
+from unique_search_proxy_client.web.logging_config import (
+    build_logging_config,
+    configure_logging,
+)
 from unique_search_proxy_client.web.monitoring import setup_prometheus
+from unique_search_proxy_client.web.settings.startup_report import (
+    log_startup_settings_report,
+)
 
-load_dotenv()
+if "pytest" not in sys.modules:
+    load_dotenv()
 
-
-def _configure_logging() -> None:
-    level_name = os.getenv("LOG_LEVEL", "INFO").upper()
-    level = getattr(logging, level_name, logging.INFO)
-    logging.basicConfig(
-        level=level,
-        format="%(levelname)s %(name)s: %(message)s",
-        force=True,
-    )
-
-
-_configure_logging()
+configure_logging()
 suppress_httpx_request_logs()
 
 _LOGGER = logging.getLogger(__name__)
@@ -52,6 +49,7 @@ logging.getLogger("uvicorn.access").addFilter(HealthCheckFilter())
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     _LOGGER.info("Starting Unique Search Proxy...")
+    log_startup_settings_report(_LOGGER)
     pool = await create_http_client_pool()
     app.state.http_client_pool = pool
     try:
@@ -95,4 +93,9 @@ app = create_app()
 if __name__ == "__main__":
     import uvicorn
 
-    uvicorn.run(app, host="0.0.0.0", port=2349)
+    uvicorn.run(
+        app,
+        host="0.0.0.0",
+        port=2349,
+        log_config=build_logging_config(),
+    )

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/unique_search_proxy_client/web/core/agent_engines/vertexai/client.py

@@ -8,6 +8,7 @@ from google.auth import load_credentials_from_dict
 from google.genai._api_client import BaseApiClient
 from google.genai.client import AsyncClient
 
+from unique_search_proxy_client.web.settings.providers.base import read_secret
 from unique_search_proxy_client.web.settings.providers.vertexai_agent import (
     vertexai_agent_credentials,
 )
@@ -22,9 +23,9 @@ def _get_base_api_client_from_service_account() -> BaseApiClient:
         "https://www.googleapis.com/auth/cloud-platform",
     ]
     service_account_info = json.loads(
-        b64decode(vertexai_agent_credentials.service_account_credentials).decode(
-            "utf-8"
-        ),
+        b64decode(
+            read_secret(vertexai_agent_credentials.service_account_credentials)
+        ).decode("utf-8"),
     )
     credentials, project_id = load_credentials_from_dict(
         service_account_info,

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/unique_search_proxy_client/web/core/client/service.py

@@ -14,6 +14,8 @@ from unique_search_proxy_client.web.settings.client import (
     ProxyConfig,
     http_client_settings,
 )
+from unique_search_proxy_client.web.settings.providers.base import read_secret
+from unique_search_proxy_client.web.settings.secret_str import read_secret_headers
 
 if TYPE_CHECKING:
     from fastapi import FastAPI
@@ -51,7 +53,8 @@ def _build_proxy_url_with_username_password(settings: HttpClientSettings) -> str
         raise ValueError("Proxy username and password are required")
     return (
         f"{settings.proxy_protocol}://"
-        f"{proxy_username}:{proxy_password}@{proxy_host}:{proxy_port}"
+        f"{read_secret(proxy_username)}:{read_secret(proxy_password)}"
+        f"@{proxy_host}:{proxy_port}"
     )
 
 
@@ -92,7 +95,7 @@ def _get_username_password_proxy_kwargs(settings: HttpClientSettings) -> ProxyCo
     )
     return ProxyConfig(
         proxy=proxy_url,
-        headers=settings.proxy_headers,
+        headers=read_secret_headers(settings.proxy_headers) or None,
         verify=settings.proxy_ssl_ca_bundle_path or True,
     )
 
@@ -104,7 +107,7 @@ def _get_ssl_tls_proxy_kwargs(settings: HttpClientSettings) -> ProxyConfig:
     return ProxyConfig(
         proxy=proxy_url,
         cert=cert_args,
-        headers=settings.proxy_headers,
+        headers=read_secret_headers(settings.proxy_headers) or None,
         verify=settings.proxy_ssl_ca_bundle_path or True,
     )
 

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/unique_search_proxy_client/web/core/crawlers/basic/service.py

@@ -14,6 +14,11 @@ from unique_search_proxy_core.schema import (
     CrawlUrlResult,
     ProxyErrorCode,
 )
+from unique_search_proxy_core.url_safety import (
+    ResolvedCrawlTarget,
+    bypass_crawl_target,
+    pinned_httpx_get_args,
+)
 
 from unique_search_proxy_client.web.core.crawlers.basic.processing import (
     ContentProcessingError,
@@ -27,6 +32,7 @@ from unique_search_proxy_client.web.core.provider_response import (
     crawl_upstream_error,
     transport_error_raw,
 )
+from unique_search_proxy_client.web.core.url_safety.gate import AllowedCrawlTarget
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -43,11 +49,30 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
 
     crawler_id = CrawlerType.BASIC.value
 
-    async def crawl(self, request: BasicCrawlRequest) -> list[CrawlUrlResult]:  # type: ignore
+    async def crawl(self, request: BasicCrawlRequest) -> list[CrawlUrlResult]:  # type: ignore[override]
+        bypass_targets = [
+            AllowedCrawlTarget(
+                display_url=url.strip(),
+                resolved=bypass_crawl_target(url),
+            )
+            for url in request.urls
+        ]
+        return await self.crawl_pinned(request, bypass_targets)
+
+    async def crawl_pinned(
+        self,
+        request: BasicCrawlRequest,  # type: ignore[valid-type]
+        allowed_targets: list[AllowedCrawlTarget],
+    ) -> list[CrawlUrlResult]:
         client = self._http_client
         if client is None:
             raise RuntimeError("HTTP client is required for Basic crawler")
 
+        display_urls = list(request.urls)
+        if len(allowed_targets) != len(display_urls):
+            msg = "allowed_targets length must match request.urls length"
+            raise ValueError(msg)
+
         timeout = request.timeout
         semaphore = asyncio.Semaphore(request.max_concurrent_requests)
         return list(
@@ -55,12 +80,13 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
                 *[
                     self._crawl_one(
                         client,
-                        url,
+                        allowed_target.display_url,
+                        resolved_target=allowed_target.resolved,
                         timeout=timeout,
                         semaphore=semaphore,
                         content_type_handlers=request.content_types.to_handlers(),
                     )
-                    for url in request.urls
+                    for allowed_target in allowed_targets
                 ],
             ),
         )
@@ -68,35 +94,38 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
     async def _crawl_one(
         self,
         client: AsyncClient,
-        url: str,
+        display_url: str,
         *,
+        resolved_target: ResolvedCrawlTarget,
         timeout: int,
         semaphore: asyncio.Semaphore,
         content_type_handlers: dict[str, ContentTypeHandlerPolicy],
     ) -> CrawlUrlResult:
-        request_url = url.strip()
+        request_url, pin_headers, extensions = pinned_httpx_get_args(resolved_target)
         async with semaphore:
-            headers = {"User-Agent": random_user_agent()}
+            headers = {"User-Agent": random_user_agent(), **pin_headers}
+
             try:
                 response = await client.get(
                     request_url,
                     headers=headers,
+                    extensions=extensions or None,
                     timeout=Timeout(timeout),
                     follow_redirects=True,
                 )
             except httpx.TimeoutException as exc:
-                _LOGGER.warning("Basic crawl timed out for %s: %s", request_url, exc)
+                _LOGGER.warning("Basic crawl timed out for %s: %s", display_url, exc)
                 return crawl_upstream_error(
-                    request_url,
+                    display_url,
                     f"Crawl timed out after {timeout}s",
                     content_type=None,
                     code=ProxyErrorCode.UPSTREAM_TIMEOUT.value,
                     raw=transport_error_raw(exc),
                 )
             except httpx.HTTPError as exc:
-                _LOGGER.warning("Basic crawl failed for %s: %s", request_url, exc)
+                _LOGGER.warning("Basic crawl failed for %s: %s", display_url, exc)
                 return crawl_upstream_error(
-                    request_url,
+                    display_url,
                     str(exc),
                     content_type=None,
                     raw=transport_error_raw(exc),
@@ -107,11 +136,11 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
             if response.is_error:
                 _LOGGER.warning(
                     "Basic crawl HTTP error for %s: %s",
-                    request_url,
+                    display_url,
                     response.status_code,
                 )
                 return crawl_upstream_error(
-                    request_url,
+                    display_url,
                     f"HTTP {response.status_code} while fetching URL",
                     content_type=content_type,
                     raw=raw_body,
@@ -120,7 +149,7 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
             content = await self._maybe_process_content(
                 raw_body,
                 content_type,
-                request_url=request_url,
+                request_url=display_url,
                 timeout=timeout,
                 content_type_handlers=content_type_handlers,
             )
@@ -128,7 +157,7 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
                 return content
 
             return CrawlUrlResult(
-                url=request_url,
+                url=display_url,
                 content=content,
                 raw=raw_body,
                 content_type=content_type,

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/unique_search_proxy_client/web/core/crawlers/firecrawl/service.py

@@ -25,6 +25,7 @@ from unique_search_proxy_client.web.core.provider_response import (
     upstream_error_message,
     upstream_response_raw,
 )
+from unique_search_proxy_client.web.settings.providers.base import read_secret
 from unique_search_proxy_client.web.settings.providers.firecrawl import (
     firecrawl_crawl_credentials as credentials,
 )
@@ -90,7 +91,7 @@ class FirecrawlCrawlerService(BaseCrawler[FirecrawlCrawlRequest]):
             response = await client.post(
                 credentials.scrape_endpoint,
                 json=body,
-                headers=_firecrawl_headers(credentials.api_key),
+                headers=_firecrawl_headers(read_secret(credentials.api_key)),
                 timeout=timeout,
             )
         except httpx.TimeoutException as exc:
@@ -149,7 +150,7 @@ class FirecrawlCrawlerService(BaseCrawler[FirecrawlCrawlRequest]):
             start_response = await client.post(
                 credentials.batch_scrape_endpoint,
                 json=body,
-                headers=_firecrawl_headers(credentials.api_key),
+                headers=_firecrawl_headers(read_secret(credentials.api_key)),
                 timeout=timeout,
             )
         except httpx.TimeoutException as exc:
@@ -200,7 +201,7 @@ class FirecrawlCrawlerService(BaseCrawler[FirecrawlCrawlRequest]):
             final_payload = await poll_batch_scrape(
                 client,
                 status_url=status_url,
-                api_key=credentials.api_key,
+                api_key=read_secret(credentials.api_key),
                 deadline=deadline,
             )
         except TimeoutError as exc:

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/unique_search_proxy_client/web/core/crawlers/jina/service.py

@@ -18,6 +18,7 @@ from unique_search_proxy_client.web.core.provider_response import (
     transport_error_raw,
     upstream_response_raw,
 )
+from unique_search_proxy_client.web.settings.providers.base import read_secret
 from unique_search_proxy_client.web.settings.providers.jina import (
     jina_crawl_credentials as credentials,
 )
@@ -70,7 +71,7 @@ class JinaCrawlerService(BaseCrawler[JinaCrawlRequest]):
 
         urls = list(request.urls)
         timeout = request.timeout
-        headers = _jina_headers(credentials.api_key)
+        headers = _jina_headers(read_secret(credentials.api_key))
         semaphore = asyncio.Semaphore(request.max_concurrent_requests)
 
         async def crawl_one(url: str) -> CrawlUrlResult:

unique_search_proxy-2026.26.0.dev11/unique_search_proxy_client/web/core/crawlers/pinned_egress.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+from pydantic import BaseModel
+from unique_search_proxy_core.schema import CrawlUrlResult
+
+from unique_search_proxy_client.web.core.url_safety.gate import AllowedCrawlTarget
+
+
+@runtime_checkable
+class PinnedEgressCrawler(Protocol):
+    """Crawlers that fetch directly and must reuse gate DNS resolution."""
+
+    async def crawl_pinned(
+        self,
+        request: BaseModel,
+        allowed_targets: list[AllowedCrawlTarget],
+    ) -> list[CrawlUrlResult]: ...
+
+
+__all__ = ["PinnedEgressCrawler"]

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/unique_search_proxy_client/web/core/crawlers/tavily/service.py

@@ -19,6 +19,7 @@ from unique_search_proxy_client.web.core.provider_response import (
     upstream_error_message,
     upstream_response_raw,
 )
+from unique_search_proxy_client.web.settings.providers.base import read_secret
 from unique_search_proxy_client.web.settings.providers.tavily import (
     tavily_crawl_credentials as credentials,
 )
@@ -132,7 +133,7 @@ class TavilyCrawlerService(BaseCrawler[TavilyCrawlRequest]):
             response = await client.post(
                 credentials.extract_endpoint,
                 json=body,
-                headers=_tavily_headers(credentials.api_key),
+                headers=_tavily_headers(read_secret(credentials.api_key)),
                 timeout=timeout,
             )
         except httpx.TimeoutException as exc:

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/unique_search_proxy_client/web/core/provider_response.py

@@ -80,6 +80,16 @@ def crawl_upstream_error(
     )
 
 
+def crawl_forbidden_target(url: str, message: str) -> CrawlUrlResult:
+    """Build a per-URL crawl failure for a URL safety policy violation."""
+    return crawl_upstream_error(
+        url,
+        message,
+        code=ProxyErrorCode.FORBIDDEN_TARGET.value,
+        content_type=None,
+    )
+
+
 def raise_for_upstream_response(
     response: httpx.Response,
     *,

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/unique_search_proxy_client/web/core/search_engines/brave/service.py

@@ -37,6 +37,7 @@ from unique_search_proxy_client.web.core.search_engines.pagination import PageRe
 from unique_search_proxy_client.web.core.search_engines.service_base import (
     SearchEngineService,
 )
+from unique_search_proxy_client.web.settings.providers.base import read_secret
 from unique_search_proxy_client.web.settings.providers.brave import (
     brave_search_credentials as credentials,
 )
@@ -71,7 +72,7 @@ class BraveSearchService(SearchEngineService[BraveSearchRequest]):
                 break
             page = await self._fetch_page(
                 request=request,
-                api_key=credentials.api_key,
+                api_key=read_secret(credentials.api_key),
                 api_endpoint=credentials.api_endpoint,
                 page=page_request,
                 timeout=timeout,

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/unique_search_proxy_client/web/core/search_engines/google/service.py

@@ -37,6 +37,7 @@ from unique_search_proxy_client.web.core.search_engines.pagination import PageRe
 from unique_search_proxy_client.web.core.search_engines.service_base import (
     SearchEngineService,
 )
+from unique_search_proxy_client.web.settings.providers.base import read_secret
 from unique_search_proxy_client.web.settings.providers.google import (
     google_search_credentials as credentials,
 )
@@ -71,7 +72,7 @@ class GoogleSearchService(SearchEngineService[GoogleSearchRequest]):
         for page_request in iter_google_page_requests(fetch_size):
             page = await self._fetch_page(
                 request=request,
-                api_key=credentials.api_key,
+                api_key=read_secret(credentials.api_key),
                 search_engine_id=search_engine_id,
                 api_endpoint=credentials.api_endpoint,
                 page=page_request,

{unique_search_proxy-2026.26.0.dev9 → unique_search_proxy-2026.26.0.dev11}/unique_search_proxy_client/web/core/search_engines/perplexity/service.py

@@ -36,6 +36,7 @@ from unique_search_proxy_client.web.core.search_engines.service_base import (
 from unique_search_proxy_client.web.settings.providers import (
     perplexity_search_credentials as credentials,
 )
+from unique_search_proxy_client.web.settings.providers.base import read_secret
 
 _LOGGER = logging.getLogger(__name__)
 _PERPLEXITY_PROVIDER_LABEL = "Perplexity Search API"
@@ -67,7 +68,7 @@ class PerplexitySearchService(SearchEngineService[PerplexitySearchRequest]):
             response = await client.post(
                 credentials.api_endpoint,
                 json=body,
-                headers=_perplexity_headers(credentials.api_key),
+                headers=_perplexity_headers(read_secret(credentials.api_key)),
                 timeout=timeout,
             )
         except httpx.TimeoutException as exc:

unique_search_proxy-2026.26.0.dev11/unique_search_proxy_client/web/core/url_safety/__init__.py

@@ -0,0 +1,13 @@
+from unique_search_proxy_client.web.core.url_safety.gate import (
+    AllowedCrawlTarget,
+    UrlSafetyGateResult,
+    apply_url_safety_gate,
+    merge_crawl_results,
+)
+
+__all__ = [
+    "AllowedCrawlTarget",
+    "UrlSafetyGateResult",
+    "apply_url_safety_gate",
+    "merge_crawl_results",
+]

unique_search_proxy-2026.26.0.dev11/unique_search_proxy_client/web/core/url_safety/gate.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from unique_search_proxy_core.schema import CrawlUrlResult
+from unique_search_proxy_core.url_safety import ResolvedCrawlTarget, UrlSafetyService
+
+from unique_search_proxy_client.web.core.provider_response import crawl_forbidden_target
+from unique_search_proxy_client.web.monitoring.metrics import record_crawl_blocked
+
+
+@dataclass(frozen=True)
+class AllowedCrawlTarget:
+    """User-facing URL paired with the validated resolution for pinned egress."""
+
+    display_url: str
+    resolved: ResolvedCrawlTarget
+
+
+@dataclass(frozen=True)
+class UrlSafetyGateResult:
+    allowed_targets: list[AllowedCrawlTarget]
+    blocked_by_index: dict[int, CrawlUrlResult]
+
+
+async def apply_url_safety_gate(urls: list[str]) -> UrlSafetyGateResult:
+    """Validate crawl URLs and partition them into allowed vs blocked targets."""
+    outcomes = await UrlSafetyService.validate_urls_individually(urls)
+    allowed_targets: list[AllowedCrawlTarget] = []
+    blocked_by_index: dict[int, CrawlUrlResult] = {}
+
+    for index, outcome in enumerate(outcomes):
+        if outcome.blocked is not None:
+            record_crawl_blocked(outcome.blocked.category)
+            blocked_by_index[index] = crawl_forbidden_target(
+                outcome.url.strip(),
+                outcome.blocked.reason,
+            )
+            continue
+
+        if outcome.resolved is None:
+            msg = "URL safety allowed a crawl target without resolved metadata"
+            raise RuntimeError(msg)
+
+        allowed_targets.append(
+            AllowedCrawlTarget(
+                display_url=outcome.url.strip(),
+                resolved=outcome.resolved,
+            ),
+        )
+
+    return UrlSafetyGateResult(
+        allowed_targets=allowed_targets,
+        blocked_by_index=blocked_by_index,
+    )
+
+
+def merge_crawl_results(
+    urls: list[str],
+    *,
+    blocked_by_index: dict[int, CrawlUrlResult],
+    crawler_results: list[CrawlUrlResult],
+) -> list[CrawlUrlResult]:
+    """Merge per-URL blocked results with crawler outcomes in request order."""
+    merged: list[CrawlUrlResult] = []
+    crawler_index = 0
+    for index, _url in enumerate(urls):
+        blocked = blocked_by_index.get(index)
+        if blocked is not None:
+            merged.append(blocked)
+            continue
+
+        merged.append(crawler_results[crawler_index])
+        crawler_index += 1
+
+    return merged

unique_search_proxy-2026.26.0.dev11/unique_search_proxy_client/web/logging_config.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import copy
+import logging
+import logging.config
+import os
+from typing import Any
+
+from uvicorn.config import LOG_LEVELS, LOGGING_CONFIG
+
+_APP_LOGGER_NAMES = (
+    "unique_search_proxy_client",
+    "unique_search_proxy_core",
+)
+
+
+def build_logging_config(log_level: str | None = None) -> dict[str, Any]:
+    """Extend Uvicorn's logging config with application loggers."""
+    level_key = (log_level or os.getenv("LOG_LEVEL", "info")).lower()
+    if level_key not in LOG_LEVELS:
+        level_key = "info"
+    level_name = level_key.upper()
+
+    config = copy.deepcopy(LOGGING_CONFIG)
+    for logger_name in _APP_LOGGER_NAMES:
+        config["loggers"][logger_name] = {
+            "handlers": ["default"],
+            "level": level_name,
+            "propagate": False,
+        }
+    config["loggers"]["uvicorn"]["level"] = level_name
+    config["loggers"]["uvicorn.error"]["level"] = level_name
+    config["loggers"]["uvicorn.access"]["level"] = level_name
+    return config
+
+
+def configure_logging(log_level: str | None = None) -> None:
+    """Configure app and Uvicorn loggers with Uvicorn's colored formatter."""
+    logging.config.dictConfig(build_logging_config(log_level))