unique-search-proxy-core 2026.26.0.dev5__tar.gz → 2026.26.0.dev7__tar.gz

{unique_search_proxy_core-2026.26.0.dev5 → unique_search_proxy_core-2026.26.0.dev7}/PKG-INFO

@@ -1,10 +1,12 @@
 Metadata-Version: 2.3
 Name: unique-search-proxy-core
-Version: 2026.26.0.dev5
+Version: 2026.26.0.dev7
 Summary: Shared Pydantic types for the Unique Search Proxy API
 Author: ThePhilAz
 Author-email: ThePhilAz <rami.azouz@philico.com>
+Requires-Dist: httpx>=0.28.0,<0.29
 Requires-Dist: pydantic>=2.12.5,<3.0.0
+Requires-Dist: pydantic-settings>=2.10.1,<3
 Requires-Dist: pyhumps>=3.8.0,<4
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown

{unique_search_proxy_core-2026.26.0.dev5 → unique_search_proxy_core-2026.26.0.dev7}/pyproject.toml

@@ -1,12 +1,14 @@
 [project]
 name = "unique-search-proxy-core"
-version = "2026.26.0.dev5"
+version = "2026.26.0.dev7"
 description = "Shared Pydantic types for the Unique Search Proxy API"
 authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
 readme = "README.md"
 requires-python = ">=3.12"
 dependencies = [
+    "httpx>=0.28.0,<0.29",
     "pydantic>=2.12.5,<3.0.0",
+    "pydantic-settings>=2.10.1,<3",
     "pyhumps>=3.8.0,<4",
 ]
 

unique_search_proxy_core-2026.26.0.dev7/unique_search_proxy_core/url_safety/__init__.py

@@ -0,0 +1,25 @@
+from unique_search_proxy_core.url_safety.egress import pinned_httpx_get_args
+from unique_search_proxy_core.url_safety.models import (
+    BlockedCrawlTarget,
+    CrawlTargetValidationError,
+    ResolvedCrawlTarget,
+    UrlSafetyOutcome,
+    bypass_crawl_target,
+)
+from unique_search_proxy_core.url_safety.service import UrlSafetyService
+from unique_search_proxy_core.url_safety.settings import (
+    UrlSafetySettings,
+    url_safety_settings,
+)
+
+__all__ = [
+    "BlockedCrawlTarget",
+    "CrawlTargetValidationError",
+    "ResolvedCrawlTarget",
+    "UrlSafetyOutcome",
+    "UrlSafetyService",
+    "UrlSafetySettings",
+    "bypass_crawl_target",
+    "pinned_httpx_get_args",
+    "url_safety_settings",
+]

unique_search_proxy_core-2026.26.0.dev7/unique_search_proxy_core/url_safety/dns.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import asyncio
+import socket
+from ipaddress import ip_address
+
+
+async def resolve_host_addresses(host: str) -> tuple[str, ...]:
+    loop = asyncio.get_running_loop()
+    resolved = await loop.run_in_executor(
+        None, lambda: socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)
+    )
+    return tuple(
+        dict.fromkeys(str(ip_address(sockaddr[0])) for _, _, _, _, sockaddr in resolved)
+    )
+
+
+def block_reason_for_resolved_addresses(
+    resolved_addresses: tuple[str, ...],
+) -> tuple[str, str] | None:
+    if not resolved_addresses:
+        return (
+            "dns",
+            "Target host could not be resolved during safety validation",
+        )
+
+    for resolved_address in resolved_addresses:
+        if not ip_address(resolved_address).is_global:
+            return (
+                "private",
+                "Target host resolves to a private or special-use IP address",
+            )
+
+    return None
+
+
+async def resolve_and_validate_host(
+    host: str,
+) -> tuple[tuple[str, ...], tuple[str, str] | None]:
+    try:
+        resolved_addresses = await resolve_host_addresses(host)
+    except socket.gaierror:
+        return (), (
+            "dns",
+            "Target host could not be resolved during safety validation",
+        )
+
+    return resolved_addresses, block_reason_for_resolved_addresses(resolved_addresses)
+
+
+async def validate_resolved_host(host: str) -> tuple[str, str] | None:
+    _, validation_error = await resolve_and_validate_host(host)
+    return validation_error

unique_search_proxy_core-2026.26.0.dev7/unique_search_proxy_core/url_safety/egress.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from unique_search_proxy_core.url_safety.models import ResolvedCrawlTarget
+
+
+def pinned_httpx_get_args(
+    target: ResolvedCrawlTarget,
+) -> tuple[str, dict[str, str], dict[str, str]]:
+    """Build httpx GET url, headers, and extensions for DNS-pinned egress."""
+    headers: dict[str, str] = {}
+    if target.host_header is not None:
+        headers["Host"] = target.host_header
+
+    extensions: dict[str, str] = {}
+    if target.sni_hostname is not None:
+        extensions["sni_hostname"] = target.sni_hostname
+
+    return target.request_url, headers, extensions
+
+
+__all__ = ["pinned_httpx_get_args"]

unique_search_proxy_core-2026.26.0.dev7/unique_search_proxy_core/url_safety/models.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from urllib.parse import urlsplit
+
+from unique_search_proxy_core.url_safety.netloc import (
+    build_netloc,
+    replace_url_host,
+)
+
+
+@dataclass(frozen=True)
+class BlockedCrawlTarget:
+    hostname: str | None
+    category: str
+    reason: str
+
+    @property
+    def display_target(self) -> str:
+        return self.hostname or "<unknown-host>"
+
+
+@dataclass(frozen=True)
+class ResolvedCrawlTarget:
+    """Carries the validated host/IP values reused when issuing crawl requests."""
+
+    normalized_url: str
+    hostname: str
+    resolved_ip: str
+    used_dns_resolution: bool
+
+    @property
+    def request_url(self) -> str:
+        if not self.used_dns_resolution and not self.resolved_ip:
+            return self.normalized_url
+        return replace_url_host(urlsplit(self.normalized_url), host=self.resolved_ip)
+
+    @property
+    def host_header(self) -> str | None:
+        if not self.used_dns_resolution:
+            return None
+
+        return build_netloc(host=self.hostname, port=urlsplit(self.normalized_url).port)
+
+    @property
+    def sni_hostname(self) -> str | None:
+        if not self.used_dns_resolution:
+            return None
+
+        if urlsplit(self.normalized_url).scheme.lower() != "https":
+            return None
+
+        return self.hostname
+
+
+@dataclass(frozen=True)
+class UrlSafetyOutcome:
+    url: str
+    resolved: ResolvedCrawlTarget | None = None
+    blocked: BlockedCrawlTarget | None = None
+
+
+class CrawlTargetValidationError(ValueError):
+    def __init__(self, blocked_targets: list[BlockedCrawlTarget]) -> None:
+        self.blocked_targets: list[BlockedCrawlTarget] = blocked_targets
+
+        details = "; ".join(
+            f"{target.display_target} ({target.reason})"
+            for target in self.blocked_targets
+        )
+        super().__init__(f"Blocked crawl target(s) due to URL safety policy: {details}")
+
+
+def bypass_crawl_target(url: str) -> ResolvedCrawlTarget:
+    return ResolvedCrawlTarget(
+        normalized_url=url.strip(),
+        hostname="",
+        resolved_ip="",
+        used_dns_resolution=False,
+    )

unique_search_proxy_core-2026.26.0.dev7/unique_search_proxy_core/url_safety/netloc.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from ipaddress import ip_address
+from urllib.parse import SplitResult, urlsplit, urlunsplit
+
+
+def extract_hostname(url: str) -> str | None:
+    normalized_url = url.strip()
+    if not normalized_url:
+        return None
+
+    hostname = urlsplit(normalized_url).hostname
+    if hostname is None:
+        return None
+
+    return hostname.rstrip(".").lower()
+
+
+def replace_url_host(parsed_url: SplitResult, *, host: str) -> str:
+    netloc = build_netloc(
+        host=host,
+        port=parsed_url.port,
+        username=parsed_url.username,
+        password=parsed_url.password,
+    )
+    return urlunsplit(
+        (
+            parsed_url.scheme,
+            netloc,
+            parsed_url.path,
+            parsed_url.query,
+            "",
+        )
+    )
+
+
+def build_netloc(
+    host: str,
+    port: int | None,
+    username: str | None = None,
+    password: str | None = None,
+) -> str:
+    credential_prefix = ""
+    if username is not None:
+        credential_prefix = username
+        if password is not None:
+            credential_prefix = f"{credential_prefix}:{password}"
+        credential_prefix = f"{credential_prefix}@"
+
+    try:
+        host_ip = ip_address(host)
+    except ValueError:
+        normalized_host = host
+    else:
+        normalized_host = (
+            f"[{host_ip.compressed}]" if host_ip.version == 6 else host_ip.compressed
+        )
+
+    if port is None:
+        return f"{credential_prefix}{normalized_host}"
+
+    return f"{credential_prefix}{normalized_host}:{port}"

unique_search_proxy_core-2026.26.0.dev7/unique_search_proxy_core/url_safety/policy.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from ipaddress import ip_address
+from urllib.parse import urlsplit
+
+from unique_search_proxy_core.url_safety.settings import url_safety_settings
+
+
+def validate_target_cheap(url: str) -> tuple[str, str] | None:
+    """Static validation only — no DNS. Returns (category, reason) if blocked, None otherwise."""
+    if not url:
+        return "empty", "URL is empty"
+
+    parsed_url = urlsplit(url)
+    if parsed_url.scheme.lower() not in url_safety_settings.allowed_schemes:
+        return "scheme", "URL scheme must be http or https"
+
+    hostname = parsed_url.hostname
+    if not hostname:
+        return "host", "URL host is missing or malformed"
+
+    normalized_host = hostname.rstrip(".").lower()
+
+    if normalized_host in url_safety_settings.metadata_hosts:
+        return "metadata", "Target points to a metadata endpoint"
+
+    if (
+        normalized_host in url_safety_settings.localhost_hosts
+        or normalized_host.endswith(".localhost")
+    ):
+        return "localhost", "Target points to a localhost host"
+
+    if normalized_host.endswith(url_safety_settings.service_suffix):
+        return "cluster", "Target points to an internal service host"
+
+    if normalized_host.endswith(url_safety_settings.cluster_local_suffix):
+        return "cluster", "Target points to an internal cluster-local host"
+
+    try:
+        target_ip = ip_address(normalized_host)
+    except ValueError:
+        pass
+    else:
+        if not target_ip.is_global:
+            return "private", "Target points to a private or special-use IP address"
+
+        return None
+
+    if "." not in normalized_host:
+        return "cluster", "Target points to a single-label internal host"
+
+    return None

unique_search_proxy_core-2026.26.0.dev7/unique_search_proxy_core/url_safety/redirect.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Awaitable, Callable
+from urllib.parse import urljoin
+
+import httpx
+
+from unique_search_proxy_core.url_safety.models import (
+    BlockedCrawlTarget,
+    CrawlTargetValidationError,
+)
+from unique_search_proxy_core.url_safety.netloc import extract_hostname
+from unique_search_proxy_core.url_safety.settings import url_safety_settings
+
+_LOGGER = logging.getLogger(__name__)
+
+_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})
+
+ValidateUrlFn = Callable[[str], Awaitable[tuple[str, str] | None]]
+
+
+async def resolve_redirect_chain(
+    url: str,
+    validate_url: ValidateUrlFn,
+) -> str:
+    """Follow HTTP 3xx redirects hop-by-hop, validating each destination.
+
+    Returns the final validated URL.
+    Raises CrawlTargetValidationError if any hop is blocked or redirect probing
+    cannot be completed (fail-closed to prevent GET-time redirect SSRF bypass).
+    """
+    current = url
+    timeout = url_safety_settings.redirect_timeout_seconds
+    async with httpx.AsyncClient(follow_redirects=False, timeout=timeout) as client:
+        for _ in range(url_safety_settings.max_redirect_hops):
+            error = await validate_url(current)
+            if error is not None:
+                category, reason = error
+                raise CrawlTargetValidationError(
+                    [
+                        BlockedCrawlTarget(
+                            hostname=extract_hostname(current),
+                            category=category,
+                            reason=reason,
+                        )
+                    ]
+                )
+
+            try:
+                resp = await client.head(current)
+            except Exception as exc:
+                _LOGGER.debug(
+                    "Redirect resolution blocked at %s due to network error: %s",
+                    current,
+                    exc,
+                )
+                raise CrawlTargetValidationError(
+                    [
+                        BlockedCrawlTarget(
+                            hostname=extract_hostname(current),
+                            category="redirect",
+                            reason=(
+                                "Unable to verify redirect chain before crawl; "
+                                "blocking to prevent redirect-based SSRF"
+                            ),
+                        )
+                    ]
+                ) from exc
+
+            if resp.status_code not in _REDIRECT_STATUS_CODES:
+                break
+
+            location = resp.headers.get("location")
+            if not location:
+                break
+
+            current = urljoin(current, location)
+
+    error = await validate_url(current)
+    if error is not None:
+        category, reason = error
+        raise CrawlTargetValidationError(
+            [
+                BlockedCrawlTarget(
+                    hostname=extract_hostname(current),
+                    category=category,
+                    reason=reason,
+                )
+            ]
+        )
+
+    return current

unique_search_proxy_core-2026.26.0.dev7/unique_search_proxy_core/url_safety/resolver.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import asyncio
+from ipaddress import ip_address
+from urllib.parse import urlsplit
+
+from unique_search_proxy_core.url_safety import dns
+from unique_search_proxy_core.url_safety.models import (
+    BlockedCrawlTarget,
+    CrawlTargetValidationError,
+    ResolvedCrawlTarget,
+)
+from unique_search_proxy_core.url_safety.netloc import extract_hostname
+from unique_search_proxy_core.url_safety.policy import validate_target_cheap
+
+
+async def resolve_crawl_target(url: str) -> ResolvedCrawlTarget:
+    """Validate and resolve a single URL, performing DNS exactly once."""
+    normalized_url = url.strip()
+
+    validation_error = validate_target_cheap(normalized_url)
+    if validation_error is not None:
+        category, reason = validation_error
+        raise CrawlTargetValidationError(
+            [
+                BlockedCrawlTarget(
+                    hostname=extract_hostname(normalized_url),
+                    category=category,
+                    reason=reason,
+                )
+            ]
+        )
+
+    parsed_url = urlsplit(normalized_url)
+    hostname = parsed_url.hostname
+    if hostname is None:
+        raise CrawlTargetValidationError(
+            [
+                BlockedCrawlTarget(
+                    hostname=None,
+                    category="host",
+                    reason="URL host is missing or malformed",
+                )
+            ]
+        )
+
+    normalized_host = hostname.rstrip(".").lower()
+
+    try:
+        target_ip = ip_address(normalized_host)
+    except ValueError:
+        resolved_addresses, validation_error = await dns.resolve_and_validate_host(
+            normalized_host
+        )
+        if validation_error is not None:
+            category, reason = validation_error
+            raise CrawlTargetValidationError(
+                [
+                    BlockedCrawlTarget(
+                        hostname=normalized_host,
+                        category=category,
+                        reason=reason,
+                    )
+                ]
+            )
+        return ResolvedCrawlTarget(
+            normalized_url=normalized_url,
+            hostname=normalized_host,
+            resolved_ip=resolved_addresses[0],
+            used_dns_resolution=True,
+        )
+
+    return ResolvedCrawlTarget(
+        normalized_url=normalized_url,
+        hostname=normalized_host,
+        resolved_ip=str(target_ip),
+        used_dns_resolution=False,
+    )
+
+
+async def resolve_crawl_targets_batch(urls: list[str]) -> list[ResolvedCrawlTarget]:
+    """Resolve and validate each URL, collecting all policy violations before raising."""
+    results: list[ResolvedCrawlTarget | BaseException] = list(
+        await asyncio.gather(
+            *[resolve_crawl_target(url) for url in urls],
+            return_exceptions=True,
+        )
+    )
+
+    blocked: list[BlockedCrawlTarget] = []
+    resolved: list[ResolvedCrawlTarget] = []
+    for result in results:
+        if isinstance(result, CrawlTargetValidationError):
+            blocked.extend(result.blocked_targets)
+        elif isinstance(result, BaseException):
+            raise result
+        else:
+            resolved.append(result)
+
+    if blocked:
+        raise CrawlTargetValidationError(blocked)
+
+    return resolved

unique_search_proxy_core-2026.26.0.dev7/unique_search_proxy_core/url_safety/service.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from ipaddress import ip_address
+from urllib.parse import urlsplit
+
+from unique_search_proxy_core.url_safety import dns, redirect, resolver
+from unique_search_proxy_core.url_safety.models import (
+    CrawlTargetValidationError,
+    ResolvedCrawlTarget,
+    UrlSafetyOutcome,
+    bypass_crawl_target,
+)
+from unique_search_proxy_core.url_safety.policy import validate_target_cheap
+from unique_search_proxy_core.url_safety.settings import url_safety_settings
+
+
+class UrlSafetyService:
+    @staticmethod
+    async def validate_urls_individually(urls: list[str]) -> list[UrlSafetyOutcome]:
+        if not url_safety_settings.enabled:
+            return [
+                UrlSafetyOutcome(url=url, resolved=bypass_crawl_target(url))
+                for url in urls
+            ]
+
+        outcomes: list[UrlSafetyOutcome] = []
+        for url in urls:
+            working_url = url.strip()
+            try:
+                if url_safety_settings.resolve_redirects:
+                    working_url = await redirect.resolve_redirect_chain(
+                        working_url,
+                        validate_url=UrlSafetyService.validate_url,
+                    )
+                resolved = await resolver.resolve_crawl_target(working_url)
+                outcomes.append(UrlSafetyOutcome(url=url, resolved=resolved))
+            except CrawlTargetValidationError as exc:
+                blocked = exc.blocked_targets[0]
+                outcomes.append(UrlSafetyOutcome(url=url, blocked=blocked))
+        return outcomes
+
+    @staticmethod
+    async def validate_batch_urls(urls: list[str]) -> list[ResolvedCrawlTarget]:
+        outcomes = await UrlSafetyService.validate_urls_individually(urls)
+        blocked_targets = [
+            outcome.blocked for outcome in outcomes if outcome.blocked is not None
+        ]
+        if blocked_targets:
+            raise CrawlTargetValidationError(blocked_targets)
+
+        return [
+            outcome.resolved for outcome in outcomes if outcome.resolved is not None
+        ]
+
+    @staticmethod
+    async def validate_url(url: str) -> tuple[str, str] | None:
+        """Full validation including async DNS check.
+
+        Returns a ``(category, reason)`` tuple if the URL should be blocked, or
+        ``None`` if it is allowed.
+        """
+        error = validate_target_cheap(url)
+        if error is not None:
+            return error
+
+        hostname = urlsplit(url).hostname
+        if hostname is None:
+            return None
+
+        normalized_host = hostname.rstrip(".").lower()
+        try:
+            ip_address(normalized_host)
+            return None
+        except ValueError:
+            pass
+
+        return await dns.validate_resolved_host(normalized_host)
+
+    @staticmethod
+    async def resolve_crawl_target(url: str) -> ResolvedCrawlTarget:
+        return await resolver.resolve_crawl_target(url)

unique_search_proxy_core-2026.26.0.dev7/unique_search_proxy_core/url_safety/settings.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from logging import getLogger
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+_LOGGER = getLogger(__name__)
+
+
+class UrlSafetySettings(BaseSettings):
+    enabled: bool = True
+    resolve_redirects: bool = True
+    allowed_schemes: list[str] = ["http", "https"]
+    localhost_hosts: list[str] = [
+        "localhost",
+        "localhost.localdomain",
+    ]
+    metadata_hosts: list[str] = [
+        "100.100.100.200",  # Alibaba Cloud
+        "169.254.169.254",  # AWS / GCP / Azure IMDS
+        "169.254.170.2",  # AWS ECS task credentials
+        "metadata.azure.internal",
+        "metadata.google.internal",
+    ]
+    cluster_local_suffix: str = ".cluster.local"
+    service_suffix: str = ".svc"
+    max_redirect_hops: int = 10
+    redirect_timeout_seconds: float = 10.0
+
+    model_config = SettingsConfigDict(
+        extra="ignore",
+        env_prefix="URL_SAFETY_",
+        case_sensitive=False,
+        frozen=True,
+    )
+
+
+url_safety_settings = UrlSafetySettings()
+_LOGGER.info("URL Safety is enabled: %s", url_safety_settings.enabled)