unique-search-proxy 2026.44.0.dev11__tar.gz → 2026.44.0.dev13__tar.gz

{unique_search_proxy-2026.44.0.dev11 → unique_search_proxy-2026.44.0.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy
-Version: 2026.44.0.dev11
+Version: 2026.44.0.dev13
 Summary: Web Search Proxy implementation
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>
@@ -20,7 +20,7 @@ Requires-Dist: certifi>=2025.11.12,<2027
 Requires-Dist: google-genai>=1.73.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
 Requires-Dist: jinja2>=3.1.6,<4
-Requires-Dist: unique-toolkit[monitoring]>=2026.44.0.dev9,<2026.44.0rc0
+Requires-Dist: unique-toolkit[monitoring]>=2026.44.0.dev10,<2026.44.0rc0
 Requires-Dist: unique-search-proxy-core>=2026.44.0.dev9,<2026.44.0rc0
 Requires-Dist: openai>=1.105.0,<3
 Requires-Python: >=3.12
@@ -120,7 +120,7 @@ Built-in providers register in `register_builtin_providers()` during `create_app
 | `POST /v1/agent-search/stream` | Same, streamed as SSE (`delta` + `done`) |
 | `POST /v1/crawl` | Crawl URLs (flat body: `crawler`, `urls`, `timeout`, …) |
 | `GET /metrics` | Prometheus (when enabled) |
-| `/docs` | Swagger UI with preset examples |
+| `/docs` | Swagger UI with preset examples (only when `EXPOSE_DOCS=true`) |
 
 OpenAPI spec: `openapi.json` (input for [SDK codegen](../unique_search_proxy_sdk/README.md)).
 
@@ -310,7 +310,7 @@ uv run python scripts/generate_sdk.py
 ## 8. Dev testing
 
 1. Start the server and configure `.env` (`.env.example` sets `REQUIRE_CONTEXT_HEADERS=false` for local use; production keeps enforcement enabled).
-2. **Swagger** — `/docs` → **Try it out** on `/v1/search` or `/v1/crawl` → pick an **Examples** preset. Context headers default to `local` in the UI.
+2. **Swagger** (if `EXPOSE_DOCS=true`) — `/docs` → **Try it out** on `/v1/search` or `/v1/crawl` → pick an **Examples** preset. Context headers default to `local` in the UI.
 3. **CLI** — same presets from the terminal (`scripts/try_presets.py` sends the same local context headers):
 
 ```bash

{unique_search_proxy-2026.44.0.dev11 → unique_search_proxy-2026.44.0.dev13}/README.md

@@ -92,7 +92,7 @@ Built-in providers register in `register_builtin_providers()` during `create_app
 | `POST /v1/agent-search/stream` | Same, streamed as SSE (`delta` + `done`) |
 | `POST /v1/crawl` | Crawl URLs (flat body: `crawler`, `urls`, `timeout`, …) |
 | `GET /metrics` | Prometheus (when enabled) |
-| `/docs` | Swagger UI with preset examples |
+| `/docs` | Swagger UI with preset examples (only when `EXPOSE_DOCS=true`) |
 
 OpenAPI spec: `openapi.json` (input for [SDK codegen](../unique_search_proxy_sdk/README.md)).
 
@@ -282,7 +282,7 @@ uv run python scripts/generate_sdk.py
 ## 8. Dev testing
 
 1. Start the server and configure `.env` (`.env.example` sets `REQUIRE_CONTEXT_HEADERS=false` for local use; production keeps enforcement enabled).
-2. **Swagger** — `/docs` → **Try it out** on `/v1/search` or `/v1/crawl` → pick an **Examples** preset. Context headers default to `local` in the UI.
+2. **Swagger** (if `EXPOSE_DOCS=true`) — `/docs` → **Try it out** on `/v1/search` or `/v1/crawl` → pick an **Examples** preset. Context headers default to `local` in the UI.
 3. **CLI** — same presets from the terminal (`scripts/try_presets.py` sends the same local context headers):
 
 ```bash

{unique_search_proxy-2026.44.0.dev11 → unique_search_proxy-2026.44.0.dev13}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy"
-version = "2026.44.0.dev11"
+version = "2026.44.0.dev13"
 description = "Web Search Proxy implementation"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"
@@ -22,7 +22,7 @@ dependencies = [
     "google-genai>=1.73.0,<2",
     "google-auth>=2.43.0,<3",
     "jinja2>=3.1.6,<4",
-    "unique-toolkit[monitoring]>=2026.44.0.dev9,<2026.44.0rc0",
+    "unique-toolkit[monitoring]>=2026.44.0.dev10,<2026.44.0rc0",
     "unique-search-proxy-core>=2026.44.0.dev9,<2026.44.0rc0",
     "openai>=1.105.0,<3",
 ]

{unique_search_proxy-2026.44.0.dev11 → unique_search_proxy-2026.44.0.dev13}/unique_search_proxy_client/web/api/v1/crawl.py

@@ -23,6 +23,7 @@ from unique_search_proxy_client.web.api.v1.openapi_examples import (
 )
 from unique_search_proxy_client.web.context import get_request_context
 from unique_search_proxy_client.web.core.client import get_http_client_registry
+from unique_search_proxy_client.web.core.crawl_limiter import get_crawl_limiter
 from unique_search_proxy_client.web.core.crawlers.factory import get_crawler_service
 from unique_search_proxy_client.web.core.crawlers.pinned_egress import (
     PinnedEgressCrawler,
@@ -93,7 +94,7 @@ async def crawl(
     body: CrawlRequest = Body(openapi_examples=CRAWL_OPENAPI_EXAMPLES),  # type: ignore[valid-type]
 ) -> CrawlResponse:
     crawler_id = body.crawler
-    timeout = body.timeout
+    timeout = min(body.timeout, request_limits_settings.max_timeout_seconds)
     started = time.perf_counter()
     _LOGGER.info(
         "crawl start crawler=%s urls=%d timeout=%ss",
@@ -104,7 +105,7 @@ async def crawl(
 
     try:
         _enforce_url_limits(body.urls)
-        async with asyncio.timeout(timeout):
+        async with get_crawl_limiter(request.app).slot(), asyncio.timeout(timeout):
             registry = get_http_client_registry(request.app)
             client = await registry.client_for(get_request_context())
             gate = await apply_url_safety_gate(
@@ -142,6 +143,7 @@ async def crawl(
             crawl_body = body.model_copy(
                 update={
                     "urls": [target.display_url for target in gate.allowed_targets],
+                    "timeout": timeout,
                 },
             )
 

{unique_search_proxy-2026.44.0.dev11 → unique_search_proxy-2026.44.0.dev13}/unique_search_proxy_client/web/api/v1/search.py

@@ -31,6 +31,9 @@ from unique_search_proxy_client.web.monitoring.metrics import (
     record_search_error,
     record_search_success,
 )
+from unique_search_proxy_client.web.settings.request_limits import (
+    request_limits_settings,
+)
 
 router = APIRouter(tags=["search"])
 _LOGGER = logging.getLogger(__name__)
@@ -63,7 +66,7 @@ async def search(
 ) -> SearchResponse:
     engine = body.engine
     engine_id = engine.value if hasattr(engine, "value") else str(engine)
-    timeout = body.timeout
+    timeout = min(body.timeout, request_limits_settings.max_timeout_seconds)
     started = time.perf_counter()
     _LOGGER.info("search start engine=%s timeout=%ss", engine_id, timeout)
 

{unique_search_proxy-2026.44.0.dev11 → unique_search_proxy-2026.44.0.dev13}/unique_search_proxy_client/web/app.py

@@ -19,6 +19,7 @@ from unique_search_proxy_client.web.core.agent_engines.bing.client import (
 from unique_search_proxy_client.web.core.client.service import (
     create_http_client_registry,
 )
+from unique_search_proxy_client.web.core.crawl_limiter import CrawlLimiter
 from unique_search_proxy_client.web.core.providers import register_builtin_providers
 from unique_search_proxy_client.web.error_handlers import register_exception_handlers
 from unique_search_proxy_client.web.logging_config import (
@@ -27,6 +28,10 @@ from unique_search_proxy_client.web.logging_config import (
 )
 from unique_search_proxy_client.web.middleware.context import RequestContextMiddleware
 from unique_search_proxy_client.web.monitoring import setup_prometheus
+from unique_search_proxy_client.web.settings.app import app_settings
+from unique_search_proxy_client.web.settings.request_limits import (
+    request_limits_settings,
+)
 from unique_search_proxy_client.web.startup_report import (
     log_startup_settings_report,
 )
@@ -61,6 +66,9 @@ async def lifespan(app: FastAPI):
     log_startup_settings_report(_LOGGER)
     registry = await create_http_client_registry()
     app.state.http_client_registry = registry
+    app.state.crawl_limiter = CrawlLimiter(
+        request_limits_settings.max_concurrent_crawls,
+    )
     try:
         await maybe_cleanup_auto_provisioned_bing_agents_on_start(registry)
         yield
@@ -85,6 +93,9 @@ def create_app() -> FastAPI:
         ),
         version="0.2.0",
         lifespan=lifespan,
+        docs_url="/docs" if app_settings.expose_docs else None,
+        redoc_url="/redoc" if app_settings.expose_docs else None,
+        openapi_url="/openapi.json" if app_settings.expose_docs else None,
         swagger_ui_parameters={
             "defaultModelsExpandDepth": 1,
             "defaultModelExpandDepth": 2,

unique_search_proxy-2026.44.0.dev13/unique_search_proxy_client/web/core/crawl_limiter.py

@@ -0,0 +1,51 @@
+"""Per-worker limit on crawl requests running at once."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+from unique_search_proxy_core.errors import RateLimitedError
+
+_RETRY_AFTER_SECONDS = 1
+
+
+class CrawlLimiter:
+    """Admits at most ``max_concurrent`` crawls and rejects the rest.
+
+    Rejecting instead of queueing keeps a burst from piling bodies up in one
+    worker; callers retry on the 429.
+    """
+
+    def __init__(self, max_concurrent: int) -> None:
+        self._max_concurrent = max_concurrent
+        self._semaphore = asyncio.Semaphore(max_concurrent)
+
+    @asynccontextmanager
+    async def slot(self) -> AsyncIterator[None]:
+        """Hold one crawl slot for the duration of the block.
+
+        Raises:
+            RateLimitedError: Every slot is taken.
+        """
+        if self._semaphore.locked():
+            raise RateLimitedError(
+                f"Crawl capacity is in use ({self._max_concurrent} concurrent "
+                "crawls per worker)",
+                retry_after_seconds=_RETRY_AFTER_SECONDS,
+            )
+        async with self._semaphore:
+            yield
+
+
+def get_crawl_limiter(app: FastAPI) -> CrawlLimiter:
+    """Return the application-owned crawl limiter."""
+    limiter = getattr(app.state, "crawl_limiter", None)
+    if limiter is None:
+        raise RuntimeError("Crawl limiter is not initialized")
+    return limiter
+
+
+__all__ = ["CrawlLimiter", "get_crawl_limiter"]

{unique_search_proxy-2026.44.0.dev11 → unique_search_proxy-2026.44.0.dev13}/unique_search_proxy_client/web/core/crawlers/basic/service.py

@@ -108,7 +108,12 @@ class BasicCrawlerService(BaseCrawler[BasicCrawlRequest]):
             raise ValueError(msg)
 
         timeout = request.timeout
-        semaphore = asyncio.Semaphore(request.max_concurrent_requests)
+        semaphore = asyncio.Semaphore(
+            min(
+                request.max_concurrent_requests,
+                request_limits_settings.max_concurrent_fetches,
+            ),
+        )
         budget = ByteBudget(request_limits_settings.max_crawl_response_bytes)
         return list(
             await asyncio.gather(

{unique_search_proxy-2026.44.0.dev11 → unique_search_proxy-2026.44.0.dev13}/unique_search_proxy_client/web/helm/generator/schema.py

@@ -129,6 +129,16 @@ def build_additional_schema(groups: tuple[HelmSettingsGroup, ...]) -> dict[str,
         for group in groups
         if group.helm_key is not None
     }
+    properties["networkPolicy"] = {
+        "properties": {
+            "enabled": {"type": "boolean", "default": True},
+            "allowDisabled": {
+                "type": "boolean",
+                "default": False,
+                "description": "Allow networkPolicy.enabled=false. Set only if another control restricts who can reach search-proxy.",
+            },
+        }
+    }
     return {
         "$schema": "https://json-schema.org/draft-07/schema#",
         "properties": properties,

{unique_search_proxy-2026.44.0.dev11 → unique_search_proxy-2026.44.0.dev13}/unique_search_proxy_client/web/helm/generator/template.py

@@ -7,6 +7,7 @@ from unique_search_proxy_client.web.helm.generator.introspect import (
 from unique_search_proxy_client.web.helm.metadata import (
     EgressDomainWildcard,
     EgressEndpointField,
+    EgressProxyHost,
 )
 from unique_search_proxy_client.web.helm.registry import HelmSettingsGroup
 
@@ -199,25 +200,83 @@ def _emit_domain_wildcard_egress(
     ]
 
 
+def _proxy_values_paths(
+    group: HelmSettingsGroup, egress: EgressProxyHost
+) -> tuple[str, str] | None:
+    host = _find_helm_field(group, egress.host_field)
+    port = _find_helm_field(group, egress.port_field)
+    if host is None or port is None:
+        return None
+    root = f".Values.{group.helm_key}"
+    return (
+        f"{root}.{host.section}.{host.helm_name}",
+        f"{root}.{port.section}.{port.helm_name}",
+    )
+
+
+def _emit_proxy_host_egress(
+    group: HelmSettingsGroup,
+    egress: EgressProxyHost,
+) -> list[str]:
+    paths = _proxy_values_paths(group, egress)
+    if paths is None:
+        return []
+
+    host, port = paths
+    ipv4_octet = r"(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})"
+    return [
+        f"{{{{ if and {host} {port} }}}}",
+        f"{{{{- $host := {host} -}}}}",
+        f"{{{{- $port := {port} -}}}}",
+        r'{{- if not (and (kindIs "string" $host) (regexMatch "^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$" $host)) -}}',
+        '{{- fail "httpClient.connection.proxyHost must be a plain hostname or IPv4 address (no scheme, port, path or IPv6 literal)." -}}',
+        "{{- end -}}",
+        '{{- if not (regexMatch "^[0-9]+$" (toString $port)) -}}',
+        '{{- fail "httpClient.connection.proxyPort must be a number." -}}',
+        "{{- else if or (lt (int $port) 1) (gt (int $port) 65535) -}}",
+        '{{- fail "httpClient.connection.proxyPort must be between 1 and 65535." -}}',
+        "{{- end -}}",
+        f'{{{{- if regexMatch "^({ipv4_octet}\\\\.){{3}}{ipv4_octet}$" $host }}}}',
+        "- toCIDR:",
+        '  - {{ printf "%s/32" $host | quote }}',
+        "{{- else }}",
+        '{{- if regexMatch "^[0-9.]+$" $host -}}',
+        '{{- fail "httpClient.connection.proxyHost is not a valid IPv4 address." -}}',
+        "{{- end }}",
+        "- toFQDNs:",
+        "  - matchName: {{ $host | quote }}",
+        "{{- end }}",
+        "  toPorts:",
+        "  - ports:",
+        "    - port: {{ $port | quote }}",
+        "      protocol: TCP",
+        "{{- end }}",
+    ]
+
+
 def _emit_egress_block(group: HelmSettingsGroup) -> list[str]:
     egress = group.egress
     if isinstance(egress, EgressEndpointField):
         return _emit_endpoint_field_egress(group, egress)
     if isinstance(egress, EgressDomainWildcard):
         return _emit_domain_wildcard_egress(group, egress)
+    if isinstance(egress, EgressProxyHost):
+        return _emit_proxy_host_egress(group, egress)
     return []
 
 
+def _egress_active_check(group: HelmSettingsGroup) -> str:
+    if isinstance(group.egress, EgressProxyHost):
+        paths = _proxy_values_paths(group, group.egress)
+        if paths is not None:
+            return f"(and {paths[0]} {paths[1]})"
+    return f"(and .Values.{group.helm_key} .Values.{group.helm_key}.enabled)"
+
+
 def _emit_has_rules_block(groups: tuple[HelmSettingsGroup, ...]) -> list[str]:
     checks = [
-        f"(and .Values.{group.helm_key} .Values.{group.helm_key}.enabled)"
-        for group in groups
-        if group.egress is not None
+        _egress_active_check(group) for group in groups if group.egress is not None
     ]
-    if not checks:
-        return ["{{/* no auto egress rules */}}"]
-    if len(checks) == 1:
-        return [f"{{{{- if {checks[0]} -}}}}true{{{{- end -}}}}"]
     joined = " ".join(checks)
     return [f"{{{{- if or {joined} -}}}}true{{{{- end -}}}}"]
 

{unique_search_proxy-2026.44.0.dev11 → unique_search_proxy-2026.44.0.dev13}/unique_search_proxy_client/web/helm/metadata.py

@@ -43,7 +43,15 @@ class EgressDomainWildcard:
     port: str = "443"
 
 
-EgressRule = EgressEndpointField | EgressDomainWildcard
+@dataclass(frozen=True)
+class EgressProxyHost:
+    """Proxy egress when host and port are set: /32 ``toCIDR`` for IPv4, else ``toFQDNs``."""
+
+    host_field: str = "proxy_host"
+    port_field: str = "proxy_port"
+
+
+EgressRule = EgressEndpointField | EgressDomainWildcard | EgressProxyHost
 """How a settings group's outbound egress hosts are derived from its settings."""
 
 _DEFAULT_EGRESS: EgressRule = EgressEndpointField()

{unique_search_proxy-2026.44.0.dev11 → unique_search_proxy-2026.44.0.dev13}/unique_search_proxy_client/web/settings/app.py

@@ -15,6 +15,7 @@ APP_ENV_PREFIX = ""
 )
 class AppSettings(BaseSettings):
     require_context_headers: bool = True
+    expose_docs: bool = False
 
 
 def get_app_settings() -> AppSettings:

{unique_search_proxy-2026.44.0.dev11 → unique_search_proxy-2026.44.0.dev13}/unique_search_proxy_client/web/settings/client.py

@@ -7,7 +7,7 @@ from unique_search_proxy_core.http_client import (
     ProxyUsernameSource,
 )
 
-from unique_search_proxy_client.web.helm.metadata import helm_settings
+from unique_search_proxy_client.web.helm.metadata import EgressProxyHost, helm_settings
 from unique_search_proxy_client.web.settings.base import get_settings
 
 HTTP_CLIENT_ENV_PREFIX = "HTTP_CLIENT_"
@@ -21,7 +21,7 @@ HTTP_CLIENT_ENV_PREFIX = "HTTP_CLIENT_"
     # the proxy fields, not by a chart-only toggle. Gating it would silently drop
     # proxy config from overlays that set it without ``enabled: true``.
     gated=False,
-    egress=None,
+    egress=EgressProxyHost(),
     env_prefix=HTTP_CLIENT_ENV_PREFIX,
     sections={
         "tuning": [

{unique_search_proxy-2026.44.0.dev11 → unique_search_proxy-2026.44.0.dev13}/unique_search_proxy_client/web/settings/request_limits.py

@@ -17,7 +17,14 @@ REQUEST_LIMITS_ENV_PREFIX = "REQUEST_LIMITS_"
     egress=None,
     env_prefix=REQUEST_LIMITS_ENV_PREFIX,
     sections={
-        "crawl": ["max_crawl_response_kib", "max_urls_per_crawl", "max_url_length"],
+        "crawl": [
+            "max_crawl_response_kib",
+            "max_urls_per_crawl",
+            "max_url_length",
+            "max_concurrent_fetches",
+            "max_concurrent_crawls",
+        ],
+        "timeouts": ["max_timeout_seconds"],
     },
 )
 class RequestLimitsSettings(BaseSettings):
@@ -59,6 +66,30 @@ class RequestLimitsSettings(BaseSettings):
             "longer URL are rejected with a 422."
         ),
     )
+    max_concurrent_fetches: int = Field(
+        default=10,
+        ge=1,
+        description=(
+            "Ceiling on a basic crawl's maxConcurrentRequests; higher values "
+            "are lowered to this."
+        ),
+    )
+    max_concurrent_crawls: int = Field(
+        default=2,
+        ge=1,
+        description=(
+            "Crawl requests one worker process runs at once. Requests beyond "
+            "this get a 429 instead of queueing."
+        ),
+    )
+    max_timeout_seconds: int = Field(
+        default=120,
+        ge=1,
+        description=(
+            "Ceiling on the timeout of crawl and search requests; longer "
+            "timeouts are lowered to this."
+        ),
+    )
 
     @property
     def max_crawl_response_bytes(self) -> int: