unique-search-proxy-core 2026.42.0.dev0__tar.gz → 2026.42.0.dev2__tar.gz

{unique_search_proxy_core-2026.42.0.dev0 → unique_search_proxy_core-2026.42.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy-core
-Version: 2026.42.0.dev0
+Version: 2026.42.0.dev2
 Summary: Shared Pydantic types for the Unique Search Proxy API
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>
@@ -8,7 +8,7 @@ Requires-Dist: httpx>=0.28.0,<0.29
 Requires-Dist: pydantic>=2.12.5,<3.0.0
 Requires-Dist: pydantic-settings>=2.10.1,<3
 Requires-Dist: pyhumps>=3.8.0,<4
-Requires-Dist: unique-toolkit[monitoring]>=2026.42.0.dev0,<2026.42.0rc0
+Requires-Dist: unique-toolkit[monitoring]>=2026.42.0.dev13,<2026.42.0rc0
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 

{unique_search_proxy_core-2026.42.0.dev0 → unique_search_proxy_core-2026.42.0.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique-search-proxy-core"
-version = "2026.42.0.dev0"
+version = "2026.42.0.dev2"
 description = "Shared Pydantic types for the Unique Search Proxy API"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"
@@ -10,7 +10,7 @@ dependencies = [
     "pydantic>=2.12.5,<3.0.0",
     "pydantic-settings>=2.10.1,<3",
     "pyhumps>=3.8.0,<4",
-    "unique-toolkit[monitoring]>=2026.42.0.dev0,<2026.42.0rc0",
+    "unique-toolkit[monitoring]>=2026.42.0.dev13,<2026.42.0rc0",
 
 ]
 

{unique_search_proxy_core-2026.42.0.dev0 → unique_search_proxy_core-2026.42.0.dev2}/unique_search_proxy_core/http_client/client.py

@@ -25,8 +25,13 @@ from unique_search_proxy_core.http_client.settings import (
     ProxyAuthMode,
     ProxySettings,
 )
+from unique_search_proxy_core.url_safety.settings import url_safety_settings
 
 _LOGGER = logging.getLogger(__name__)
+_PROXY_REMEDIATION = (
+    "Either set URL_SAFETY_ENABLED=true to enable application URL-safety checks, "
+    "or configure an authenticated corporate proxy for all egress."
+)
 
 
 @dataclass(frozen=True)
@@ -207,6 +212,51 @@ def async_client_factory(
             )
 
 
+def _validate_url_safety_proxy_configuration(proxy: ProxySettings) -> None:
+    """Reject unsafe egress before the registry can return an HTTP client."""
+    if url_safety_settings.enabled:
+        return
+
+    if (
+        not proxy.proxy_host
+        or proxy.proxy_port is None
+        or not 1 <= proxy.proxy_port <= 65535
+    ):
+        raise ValueError(
+            "URL_SAFETY_ENABLED=false disables application URL-safety checks, "
+            "but no usable corporate proxy endpoint is configured: proxy_host "
+            "must be non-empty and proxy_port must be between 1 and 65535. "
+            + _PROXY_REMEDIATION,
+        )
+
+    uses_authenticated_proxy = proxy.proxy_auth_mode != "none"
+    uses_proxy_for_every_user = (
+        proxy.proxy_username_source == "user_metadata"
+        and not proxy.per_user_proxy_company_ids
+    )
+    if not uses_authenticated_proxy and not uses_proxy_for_every_user:
+        raise ValueError(
+            "URL_SAFETY_ENABLED=false disables application URL-safety checks, "
+            "but the current proxy settings do not route every request through "
+            "an authenticated corporate proxy. Direct or partially proxied "
+            "egress is incompatible. " + _PROXY_REMEDIATION,
+        )
+
+    if proxy.proxy_auth_mode == "ssl_tls" and proxy.proxy_ssl_cert_path is None:
+        raise ValueError(
+            "URL_SAFETY_ENABLED=false disables application URL-safety checks, "
+            "but SSL/TLS proxy authentication is missing proxy_ssl_cert_path. "
+            + _PROXY_REMEDIATION,
+        )
+
+    _LOGGER.warning(
+        "Application URL safety checks are disabled because "
+        "URL_SAFETY_ENABLED=false. All egress must use a configured corporate "
+        "proxy. Note: the corporate proxy URL-safety policy is not verified by "
+        "the application."
+    )
+
+
 class HttpClientRegistry:
     """Bounded LRU of httpx clients keyed by proxy credentials.
 
@@ -223,6 +273,8 @@ class HttpClientRegistry:
         *,
         fixed_client: AsyncClient | None = None,
     ) -> None:
+        if fixed_client is None:
+            _validate_url_safety_proxy_configuration(settings)
         self._settings = settings
         self._resolver = resolver
         self._fixed_client = fixed_client

{unique_search_proxy_core-2026.42.0.dev0 → unique_search_proxy_core-2026.42.0.dev2}/unique_search_proxy_core/url_safety/__init__.py

@@ -1,4 +1,7 @@
-from unique_search_proxy_core.url_safety.egress import pinned_httpx_get_args
+from unique_search_proxy_core.url_safety.egress import (
+    pinned_httpx_get_args,
+    safe_pinned_httpx_get,
+)
 from unique_search_proxy_core.url_safety.models import (
     BlockedCrawlTarget,
     CrawlTargetValidationError,
@@ -21,5 +24,6 @@ __all__ = [
     "UrlSafetySettings",
     "bypass_crawl_target",
     "pinned_httpx_get_args",
+    "safe_pinned_httpx_get",
     "url_safety_settings",
 ]

unique_search_proxy_core-2026.42.0.dev2/unique_search_proxy_core/url_safety/egress.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from urllib.parse import urlsplit
+
+import httpx
+
+from unique_search_proxy_core.url_safety.models import (
+    BlockedCrawlTarget,
+    CrawlTargetValidationError,
+    ResolvedCrawlTarget,
+    bypass_crawl_target,
+)
+from unique_search_proxy_core.url_safety.redirect import redirect_target_url
+from unique_search_proxy_core.url_safety.resolver import resolve_crawl_target
+
+
+def pinned_httpx_get_args(
+    target: ResolvedCrawlTarget,
+) -> tuple[str, dict[str, str], dict[str, str]]:
+    """Build httpx GET url, headers, and extensions for DNS-pinned egress."""
+    headers: dict[str, str] = {}
+    if target.host_header is not None:
+        headers["Host"] = target.host_header
+
+    extensions: dict[str, str] = {}
+    if target.sni_hostname is not None:
+        extensions["sni_hostname"] = target.sni_hostname
+
+    return target.request_url, headers, extensions
+
+
+async def safe_pinned_httpx_get(
+    client: httpx.AsyncClient,
+    target: ResolvedCrawlTarget,
+    *,
+    headers: Mapping[str, str] | None = None,
+    timeout: float | httpx.Timeout,
+    max_redirect_hops: int,
+    enforce_url_safety: bool,
+) -> httpx.Response:
+    """GET a target, applying the original target's safety mode to redirects."""
+    if max_redirect_hops < 0:
+        raise ValueError("max_redirect_hops must be non-negative")
+
+    current_target = target
+    for redirects_followed in range(max_redirect_hops + 1):
+        request_url, pin_headers, extensions = pinned_httpx_get_args(current_target)
+        response = await client.get(
+            request_url,
+            headers={**(headers or {}), **pin_headers},
+            extensions=extensions or None,
+            timeout=timeout,
+            follow_redirects=False,
+        )
+
+        try:
+            next_url = redirect_target_url(response, current_target.normalized_url)
+            if next_url is not None:
+                parsed_next_url = urlsplit(next_url)
+                _ = parsed_next_url.hostname
+                _ = parsed_next_url.port
+        except ValueError as exc:
+            raise CrawlTargetValidationError(
+                [
+                    BlockedCrawlTarget(
+                        hostname=current_target.hostname or None,
+                        category="redirect",
+                        reason="Redirect target URL is missing or malformed",
+                    )
+                ]
+            ) from exc
+        if next_url is None:
+            return response
+
+        if redirects_followed == max_redirect_hops:
+            raise CrawlTargetValidationError(
+                [
+                    BlockedCrawlTarget(
+                        hostname=current_target.hostname or None,
+                        category="redirect",
+                        reason="Maximum redirect hop count exceeded",
+                    )
+                ]
+            )
+
+        current_target = (
+            await resolve_crawl_target(next_url)
+            if enforce_url_safety
+            else bypass_crawl_target(next_url)
+        )
+
+    raise AssertionError("redirect loop must return or raise")
+
+
+__all__ = ["pinned_httpx_get_args", "safe_pinned_httpx_get"]

{unique_search_proxy_core-2026.42.0.dev0 → unique_search_proxy_core-2026.42.0.dev2}/unique_search_proxy_core/url_safety/redirect.py

@@ -28,6 +28,18 @@ _REDIRECT_PROBE_USER_AGENT = (
 ValidateUrlFn = Callable[[str], Awaitable[tuple[str, str] | None]]
 
 
+def redirect_target_url(response: httpx.Response, current_url: str) -> str | None:
+    """Return the next absolute redirect URL, or ``None`` when not followable."""
+    if response.status_code not in _REDIRECT_STATUS_CODES:
+        return None
+
+    location = response.headers.get("location")
+    if not location:
+        return None
+
+    return urljoin(current_url, location)
+
+
 @asynccontextmanager
 async def _redirect_probe_client(
     http_client: httpx.AsyncClient | None,
@@ -105,14 +117,11 @@ async def resolve_redirect_chain(
                     ]
                 ) from exc
 
-            if resp.status_code not in _REDIRECT_STATUS_CODES:
-                break
-
-            location = resp.headers.get("location")
-            if not location:
+            redirect_url = redirect_target_url(resp, current)
+            if redirect_url is None:
                 break
 
-            current = urljoin(current, location)
+            current = redirect_url
 
     error = await validate_url(current)
     if error is not None:

{unique_search_proxy_core-2026.42.0.dev0 → unique_search_proxy_core-2026.42.0.dev2}/unique_search_proxy_core/url_safety/settings.py

@@ -1,14 +1,19 @@
 from __future__ import annotations
 
-from logging import getLogger
-
+from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
-_LOGGER = getLogger(__name__)
-
 
 class UrlSafetySettings(BaseSettings):
-    enabled: bool = True
+    enabled: bool = Field(
+        default=True,
+        description=(
+            "Whether the application enforces URL-safety checks. Setting this to "
+            "false bypasses application checks and requires every request to use "
+            "a configured corporate proxy; the application does not verify the "
+            "proxy's URL-safety policy."
+        ),
+    )
     resolve_redirects: bool = True
     allowed_schemes: list[str] = ["http", "https"]
     localhost_hosts: list[str] = [
@@ -36,4 +41,3 @@ class UrlSafetySettings(BaseSettings):
 
 
 url_safety_settings = UrlSafetySettings()
-_LOGGER.info("URL Safety is enabled: %s", url_safety_settings.enabled)

unique_search_proxy_core-2026.42.0.dev0/unique_search_proxy_core/url_safety/egress.py

@@ -1,21 +0,0 @@
-from __future__ import annotations
-
-from unique_search_proxy_core.url_safety.models import ResolvedCrawlTarget
-
-
-def pinned_httpx_get_args(
-    target: ResolvedCrawlTarget,
-) -> tuple[str, dict[str, str], dict[str, str]]:
-    """Build httpx GET url, headers, and extensions for DNS-pinned egress."""
-    headers: dict[str, str] = {}
-    if target.host_header is not None:
-        headers["Host"] = target.host_header
-
-    extensions: dict[str, str] = {}
-    if target.sni_hostname is not None:
-        extensions["sni_hostname"] = target.sni_hostname
-
-    return target.request_url, headers, extensions
-
-
-__all__ = ["pinned_httpx_get_args"]