unique_web_search 2026.22.0.dev0__tar.gz → 2026.22.0.dev2__tar.gz

{unique_web_search-2026.22.0.dev0 → unique_web_search-2026.22.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-web-search
-Version: 2026.22.0.dev0
+Version: 2026.22.0.dev2
 Summary: 
 Author: Andreas Hauri, Gustav Hartz, Rami Azouz
 Author-email: Andreas Hauri <andreas@unique.ch>, Gustav Hartz <gustav.hartz.ext@unique.ch>, Rami Azouz <rami.ext@unique.ch>
@@ -17,7 +17,7 @@ Requires-Dist: tavily-python>=0.7.11,<1
 Requires-Dist: unidecode>=1.4.0,<2
 Requires-Dist: azure-ai-projects>=1.0.0,<2
 Requires-Dist: azure-identity>=1.25.0,<2
-Requires-Dist: unique-toolkit[monitoring]>=2026.22.0.dev0,<2026.22.0rc0
+Requires-Dist: unique-toolkit[monitoring]>=2026.22.0.dev2,<2026.22.0rc0
 Requires-Dist: azure-core>=1.36.0,<2
 Requires-Dist: google-cloud-aiplatform>=1.128.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
@@ -26,6 +26,7 @@ Requires-Dist: rapidfuzz>=3.0.0,<4
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: certifi>=2025.11.12
 Requires-Dist: jinja2>=3.1.6
+Requires-Dist: playwright>=1.58.0
 Requires-Python: >=3.12, <4.0
 Description-Content-Type: text/markdown
 

{unique_web_search-2026.22.0.dev0 → unique_web_search-2026.22.0.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_web_search"
-version = "2026.22.0.dev0"
+version = "2026.22.0.dev2"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12,<4.0"
@@ -23,7 +23,7 @@ dependencies = [
     "unidecode>=1.4.0,<2",
     "azure-ai-projects>=1.0.0,<2",
     "azure-identity>=1.25.0,<2",
-    "unique-toolkit[monitoring]>=2026.22.0.dev0,<2026.22.0rc0",
+    "unique-toolkit[monitoring]>=2026.22.0.dev2,<2026.22.0rc0",
     "azure-core>=1.36.0,<2",
     "google-cloud-aiplatform>=1.128.0,<2",
     "google-auth>=2.43.0,<3",
@@ -32,6 +32,7 @@ dependencies = [
     "httpx>=0.28.1",
     "certifi>=2025.11.12",
     "jinja2>=3.1.6",
+    "playwright>=1.58.0",
 ]
 
 [build-system]

{unique_web_search-2026.22.0.dev0 → unique_web_search-2026.22.0.dev2}/src/unique_web_search/metrics.py

@@ -39,3 +39,8 @@ crawl_duration = m.histogram("crawl_duration_seconds", "Crawl latency", ["crawle
 crawl_errors = m.counter(
     "crawl_errors_total", "Crawl failures", ["crawler", "error_type"]
 )
+crawl_blocked = m.counter(
+    "crawl_blocked_total",
+    "Crawl targets blocked by URL safety policy",
+    ["reason_category"],
+)

{unique_web_search-2026.22.0.dev0 → unique_web_search-2026.22.0.dev2}/src/unique_web_search/services/crawlers/base.py

@@ -1,3 +1,4 @@
+import asyncio
 from abc import ABC, abstractmethod
 from enum import StrEnum
 from typing import Generic, TypeVar
@@ -11,6 +12,11 @@ from unique_web_search.services.helpers import (
     clean_model_title_generator,
     experimental_model_title_generator,
 )
+from unique_web_search.services.url_safety import (
+    resolve_redirect_chain,
+    validate_crawl_urls,
+)
+from unique_web_search.settings import env_settings
 
 
 class CrawlerType(StrEnum):
@@ -50,5 +56,13 @@ class BaseCrawler(ABC, Generic[CrawlerConfig]):
     def __init__(self, config: CrawlerConfig):
         self.config = config
 
+    async def crawl(self, urls: list[str]) -> list[str]:
+        urls = await validate_crawl_urls(urls)
+        if env_settings.url_safety_resolve_redirects:
+            urls = list(
+                await asyncio.gather(*[resolve_redirect_chain(u) for u in urls])
+            )
+        return await self._crawl(urls)
+
     @abstractmethod
-    async def crawl(self, urls: list[str]) -> list[str]: ...
+    async def _crawl(self, urls: list[str]) -> list[str]: ...

{unique_web_search-2026.22.0.dev0 → unique_web_search-2026.22.0.dev2}/src/unique_web_search/services/crawlers/basic.py

@@ -7,6 +7,7 @@ import timeout_decorator
 from httpx import AsyncClient, Timeout
 from markdownify import markdownify
 from pydantic import Field
+from typing_extensions import override
 
 from unique_web_search.services.client.proxy_config import async_client
 from unique_web_search.services.crawlers.base import (
@@ -15,6 +16,10 @@ from unique_web_search.services.crawlers.base import (
     CrawlerType,
 )
 from unique_web_search.services.crawlers.utils import get_random_user_agent
+from unique_web_search.services.url_safety import (
+    CrawlTargetValidationError,
+    resolve_crawl_target,
+)
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -49,18 +54,29 @@ class BasicCrawler(BaseCrawler[BasicCrawlerConfig]):
     def __init__(self, config: BasicCrawlerConfig):
         super().__init__(config)
 
-        self.semaphore = asyncio.Semaphore(self.config.max_concurrent_requests)
+        self.semaphore: asyncio.Semaphore = asyncio.Semaphore(
+            self.config.max_concurrent_requests
+        )
 
     # TODO: Find a good way for tracking
     # @track(
     #     tags=["basic", "scrape"],
     # )
-    async def crawl(self, urls: list[str]) -> list[str]:
+    @override
+    async def _crawl(self, urls: list[str]) -> list[str]:
         async with async_client(timeout=Timeout(self.config.timeout)) as client:
             tasks = [self._crawl_url_with_client(client, url) for url in urls]
             results = await asyncio.gather(*tasks, return_exceptions=True)
 
-            markdowns = []
+            validation_errors = [
+                result
+                for result in results
+                if isinstance(result, CrawlTargetValidationError)
+            ]
+            if validation_errors:
+                raise validation_errors[0]
+
+            markdowns: list[str] = []
             for result in results:
                 if isinstance(result, BaseException):
                     markdowns.append(
@@ -78,11 +94,22 @@ class BasicCrawler(BaseCrawler[BasicCrawlerConfig]):
         if self._is_url_blacklisted(url):
             return "Unable to crawl URL due to blacklisted pattern"
 
-        response = await client.get(url, headers=headers)
+        (
+            request_url,
+            request_headers,
+            request_extensions,
+        ) = await self._build_request_target(url, headers)
+        response = await client.get(
+            request_url,
+            headers=request_headers,
+            extensions=request_extensions or None,
+        )
 
-        response.raise_for_status()
+        _ = response.raise_for_status()
 
-        content_type = response.headers.get("content-type", "").lower().split(";")[0]
+        content_type = (
+            str(response.headers.get("content-type", "")).lower().split(";")[0]
+        )
 
         if self._content_type_not_allowed(content_type):
             return f"Content type {content_type} is not allowed"
@@ -101,6 +128,22 @@ class BasicCrawler(BaseCrawler[BasicCrawlerConfig]):
     def _content_type_not_allowed(self, content_type: str) -> bool:
         return content_type in self.config.unwanted_content_types
 
+    async def _build_request_target(
+        self,
+        url: str,
+        headers: dict[str, str],
+    ) -> tuple[str, dict[str, str], dict[str, str]]:
+        resolved_target = await resolve_crawl_target(url)
+
+        request_headers = dict(headers)
+        request_extensions: dict[str, str] = {}
+        if resolved_target.host_header is not None:
+            request_headers["Host"] = resolved_target.host_header
+        if resolved_target.sni_hostname is not None:
+            request_extensions["sni_hostname"] = resolved_target.sni_hostname
+
+        return resolved_target.request_url, request_headers, request_extensions
+
 
 def _markdownify_html_with_timeout(content: str, timeout: float) -> str:
     @timeout_decorator.timeout(timeout)

{unique_web_search-2026.22.0.dev0 → unique_web_search-2026.22.0.dev2}/src/unique_web_search/services/crawlers/crawl4ai.py

@@ -2,6 +2,8 @@ import logging
 from enum import StrEnum
 from typing import Any, Literal
 
+from crawl4ai.async_webcrawler import AsyncPlaywrightCrawlerStrategy
+from playwright.async_api import Page, Request, Route
 from pydantic import BaseModel, Field
 from unique_toolkit.agentic.tools.config import get_configuration_dict
 
@@ -11,10 +13,45 @@ from unique_web_search.services.crawlers.base import (
     CrawlerType,
 )
 from unique_web_search.services.crawlers.utils import get_random_user_agent
+from unique_web_search.services.url_safety import validate_url
 
 _LOGGER = logging.getLogger(__name__)
 
 
+async def _ssrf_guard_hook(page: Page, **kwargs: object) -> None:
+    """Playwright ``before_goto`` hook that installs a route interceptor.
+
+    Registers a ``page.route`` handler that validates **every** network request
+    Chromium makes (navigation, sub-resources, JS ``fetch``) against the shared
+    url_safety rules.  Requests to internal/private targets are aborted before
+    any TCP connection is opened, closing bypass vectors A–D and sub-resource
+    SSRF that pre-flight URL validation cannot see.
+
+    crawl4ai calls this hook as::
+
+        hook(page, context=context, url=url, config=config)
+
+    so all arguments after ``page`` are captured via ``**kwargs``.
+    """
+
+    async def _route_handler(route: Route, request: Request) -> None:
+        req_url = request.url
+        error = await validate_url(req_url)
+        if error is not None:
+            category, reason = error
+            _LOGGER.warning(
+                "Crawl4AI blocked internal request: %s — %s (%s)",
+                req_url,
+                reason,
+                category,
+            )
+            await route.abort("blockedbyclient")
+            return
+        await route.continue_()
+
+    await page.route("**/*", _route_handler)
+
+
 class DisplayMode(StrEnum):
     DETAILED = "DETAILED"
     AGGREGATED = "AGGREGATED"
@@ -164,7 +201,7 @@ class Crawl4AiCrawler(BaseCrawler[Crawl4AiCrawlerConfig]):
     # @track(
     #     tags=["crawl4ai", "scrape"],
     # )
-    async def crawl(self, urls: list[str]) -> list[str]:
+    async def _crawl(self, urls: list[str]) -> list[str]:
         # Lazy import of crawl4ai - only import when actually needed
         from crawl4ai import (
             AsyncWebCrawler,
@@ -234,8 +271,18 @@ class Crawl4AiCrawler(BaseCrawler[Crawl4AiCrawlerConfig]):
             rate_limiter=rate_limiter,
         )
 
+        crawler_strategy = AsyncPlaywrightCrawlerStrategy(
+            browser_config=browser_config,
+            hooks={
+                "before_goto": _ssrf_guard_hook,
+            },
+        )
+
         _LOGGER.info(f"Crawling {len(urls)} URLs with Crawl4AiCrawler")
-        async with AsyncWebCrawler(config=browser_config) as crawler:
+
+        async with AsyncWebCrawler(
+            crawler_strategy=crawler_strategy, config=browser_config
+        ) as crawler:
             crawler_results = await crawler.arun_many(
                 urls, config=run_config, dispatcher=dispatcher
             )

{unique_web_search-2026.22.0.dev0 → unique_web_search-2026.22.0.dev2}/src/unique_web_search/services/crawlers/firecrawl.py

@@ -24,7 +24,7 @@ class FirecrawlCrawler(BaseCrawler[FirecrawlCrawlerConfig]):
     # @track(
     #     tags=["firecrawl", "scrape"],
     # )
-    async def crawl(self, urls: list[str]) -> list[str]:
+    async def _crawl(self, urls: list[str]) -> list[str]:
         api_key = get_firecrawl_search_settings().api_key
         assert api_key is not None, "Firecrawl API key is not configured"
 

{unique_web_search-2026.22.0.dev0 → unique_web_search-2026.22.0.dev2}/src/unique_web_search/services/crawlers/jina.py

@@ -52,7 +52,7 @@ class JinaCrawler(BaseCrawler[JinaCrawlerConfig]):
     # @track(
     #     tags=["jina", "scrape"],
     # )
-    async def crawl(self, urls: list[str]) -> list[str]:
+    async def _crawl(self, urls: list[str]) -> list[str]:
         jina_settings = get_jina_search_settings()
         api_key = jina_settings.api_key
         assert api_key is not None, "Jina API key is not configured"

{unique_web_search-2026.22.0.dev0 → unique_web_search-2026.22.0.dev2}/src/unique_web_search/services/crawlers/tavily.py

@@ -24,7 +24,7 @@ class TavilyCrawler(BaseCrawler[TavilyCrawlerConfig]):
     # @track(
     #     tags=["tavily", "scrape"],
     # )
-    async def crawl(self, urls: list[str]) -> list[str]:
+    async def _crawl(self, urls: list[str]) -> list[str]:
         api_key = get_tavily_search_settings().api_key
         assert api_key is not None, "Tavily API key is not configured"
 

{unique_web_search-2026.22.0.dev0 → unique_web_search-2026.22.0.dev2}/src/unique_web_search/services/executors/v2/executor.py

@@ -196,7 +196,6 @@ class WebSearchV2Executor(BaseWebSearchExecutor[WebSearchPlan]):
         crawler = self.crawler_service.config.crawler_type.value
         time_start = time()
         _LOGGER.info(f"Company {self.company_id} Crawling with {self.crawler_service}")
-
         with metric_scope(crawl_duration, crawl_errors, crawler=crawler):
             results = await self.crawler_service.crawl([step.query_or_url])
         await self._message_log_callback.log_web_search_results(

{unique_web_search-2026.22.0.dev0 → unique_web_search-2026.22.0.dev2}/src/unique_web_search/services/executors/v3/executor.py

@@ -182,7 +182,6 @@ class WebSearchV3Executor(BaseWebSearchExecutor[WebSearchV3ToolParameters]):
         crawler = self.crawler_service.config.crawler_type.value
         time_start = time()
         _LOGGER.info("Company %s Crawling %s URLs", self.company_id, len(urls))
-
         await self._message_log_callback.log_queries(urls)
         with metric_scope(crawl_duration, crawl_errors, crawler=crawler):
             contents = await self.crawler_service.crawl(urls)

unique_web_search-2026.22.0.dev2/src/unique_web_search/services/url_safety.py

@@ -0,0 +1,458 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import socket
+from dataclasses import dataclass
+from ipaddress import ip_address
+from urllib.parse import SplitResult, urljoin, urlsplit, urlunsplit
+
+import httpx
+
+from unique_web_search.metrics import crawl_blocked
+from unique_web_search.settings import env_settings
+
+_LOGGER = logging.getLogger(__name__)
+
+_MAX_REDIRECT_HOPS = 10
+_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})
+
+
+def _allowed_schemes() -> frozenset[str]:
+    return frozenset(env_settings.url_safety_allowed_schemes)
+
+
+def _localhost_hosts() -> frozenset[str]:
+    return frozenset(env_settings.url_safety_localhost_hosts)
+
+
+def _metadata_hosts() -> frozenset[str]:
+    return frozenset(env_settings.url_safety_metadata_hosts)
+
+
+def _cluster_local_suffix() -> str:
+    return env_settings.url_safety_cluster_local_suffix
+
+
+def _service_suffix() -> str:
+    return env_settings.url_safety_service_suffix
+
+
+@dataclass(frozen=True)
+class BlockedCrawlTarget:
+    hostname: str | None
+    category: str
+    reason: str
+
+    @property
+    def display_target(self) -> str:
+        return self.hostname or "<unknown-host>"
+
+
+@dataclass(frozen=True)
+class ResolvedCrawlTarget:
+    """Carries the validated host/IP values reused when issuing crawl requests."""
+
+    normalized_url: str
+    hostname: str
+    resolved_ip: str
+    used_dns_resolution: bool
+
+    @property
+    def request_url(self) -> str:
+        return _replace_url_host(urlsplit(self.normalized_url), host=self.resolved_ip)
+
+    @property
+    def host_header(self) -> str | None:
+        if not self.used_dns_resolution:
+            return None
+
+        return _build_netloc(
+            host=self.hostname, port=urlsplit(self.normalized_url).port
+        )
+
+    @property
+    def sni_hostname(self) -> str | None:
+        if not self.used_dns_resolution:
+            return None
+
+        if urlsplit(self.normalized_url).scheme.lower() != "https":
+            return None
+
+        return self.hostname
+
+
+class CrawlTargetValidationError(ValueError):
+    def __init__(self, blocked_targets: list[BlockedCrawlTarget]):
+        self.blocked_targets: list[BlockedCrawlTarget] = blocked_targets
+
+        for target in blocked_targets:
+            crawl_blocked.labels(reason_category=target.category).inc()
+
+        details = "; ".join(
+            f"{target.display_target} ({target.reason})"
+            for target in self.blocked_targets
+        )
+        super().__init__(f"Blocked crawl target(s) due to URL safety policy: {details}")
+
+
+async def validate_crawl_urls(urls: list[str]) -> list[str]:
+    normalized_urls: list[str] = []
+    blocked_targets: list[BlockedCrawlTarget] = []
+
+    for raw_url in urls:
+        normalized_url = raw_url.strip()
+        validation_error = _validate_crawl_target_cheap(normalized_url)
+        if validation_error is not None:
+            category, reason = validation_error
+            blocked_targets.append(
+                BlockedCrawlTarget(
+                    hostname=_extract_hostname(normalized_url or raw_url),
+                    category=category,
+                    reason=reason,
+                )
+            )
+        else:
+            normalized_urls.append(normalized_url)
+
+    if blocked_targets:
+        raise CrawlTargetValidationError(blocked_targets)
+
+    # Validate hostnames via DNS concurrently to avoid blocking the event loop
+    urls_needing_dns = [
+        (url, h)
+        for url in normalized_urls
+        if (h := _hostname_requiring_dns(url)) is not None
+    ]
+    if urls_needing_dns:
+        dns_results = await asyncio.gather(
+            *[_validate_resolved_host(h) for _, h in urls_needing_dns]
+        )
+        dns_blocked = [
+            BlockedCrawlTarget(
+                hostname=_extract_hostname(url),
+                category=category,
+                reason=reason,
+            )
+            for (url, _), dns_error in zip(urls_needing_dns, dns_results)
+            if dns_error is not None
+            for category, reason in [dns_error]
+        ]
+        if dns_blocked:
+            raise CrawlTargetValidationError(dns_blocked)
+
+    return normalized_urls
+
+
+async def resolve_crawl_target(url: str) -> ResolvedCrawlTarget:
+    """Validate and resolve a single URL, performing DNS exactly once."""
+    normalized_url = url.strip()
+
+    validation_error = _validate_crawl_target_cheap(normalized_url)
+    if validation_error is not None:
+        category, reason = validation_error
+        raise CrawlTargetValidationError(
+            [
+                BlockedCrawlTarget(
+                    hostname=_extract_hostname(normalized_url),
+                    category=category,
+                    reason=reason,
+                )
+            ]
+        )
+
+    parsed_url = urlsplit(normalized_url)
+    hostname = parsed_url.hostname
+    if hostname is None:
+        raise CrawlTargetValidationError(
+            [
+                BlockedCrawlTarget(
+                    hostname=None,
+                    category="host",
+                    reason="URL host is missing or malformed",
+                )
+            ]
+        )
+
+    normalized_host = hostname.rstrip(".").lower()
+
+    try:
+        target_ip = ip_address(normalized_host)
+    except ValueError:
+        resolved_addresses, validation_error = await _resolve_and_validate_host(
+            normalized_host
+        )
+        if validation_error is not None:
+            category, reason = validation_error
+            raise CrawlTargetValidationError(
+                [
+                    BlockedCrawlTarget(
+                        hostname=normalized_host,
+                        category=category,
+                        reason=reason,
+                    )
+                ]
+            )
+        return ResolvedCrawlTarget(
+            normalized_url=normalized_url,
+            hostname=normalized_host,
+            resolved_ip=resolved_addresses[0],
+            used_dns_resolution=True,
+        )
+
+    return ResolvedCrawlTarget(
+        normalized_url=normalized_url,
+        hostname=normalized_host,
+        resolved_ip=str(target_ip),
+        used_dns_resolution=False,
+    )
+
+
+def _validate_crawl_target_cheap(url: str) -> tuple[str, str] | None:
+    """Static validation only — no DNS. Returns (category, reason) if blocked, None otherwise."""
+    if not url:
+        return "empty", "URL is empty"
+
+    parsed_url = urlsplit(url)
+    if parsed_url.scheme.lower() not in _allowed_schemes():
+        return "scheme", "URL scheme must be http or https"
+
+    hostname = parsed_url.hostname
+    if not hostname:
+        return "host", "URL host is missing or malformed"
+
+    normalized_host = hostname.rstrip(".").lower()
+
+    if normalized_host in _metadata_hosts():
+        return "metadata", "Target points to a metadata endpoint"
+
+    if normalized_host in _localhost_hosts() or normalized_host.endswith(".localhost"):
+        return "localhost", "Target points to a localhost host"
+
+    if normalized_host.endswith(_service_suffix()):
+        return "cluster", "Target points to an internal service host"
+
+    if normalized_host.endswith(_cluster_local_suffix()):
+        return "cluster", "Target points to an internal cluster-local host"
+
+    try:
+        target_ip = ip_address(normalized_host)
+    except ValueError:
+        pass
+    else:
+        if not target_ip.is_global:
+            return "private", "Target points to a private or special-use IP address"
+
+        return None
+
+    if "." not in normalized_host:
+        return "cluster", "Target points to a single-label internal host"
+
+    return None
+
+
+async def validate_url(url: str) -> tuple[str, str] | None:
+    """Full validation including async DNS check.
+
+    Returns a ``(category, reason)`` tuple if the URL should be blocked, or
+    ``None`` if it is allowed.  Callers that need to raise should wrap the
+    result in :class:`CrawlTargetValidationError`.
+    """
+    error = _validate_crawl_target_cheap(url)
+    if error is not None:
+        return error
+
+    hostname = urlsplit(url).hostname
+    if hostname is None:
+        return None
+
+    normalized_host = hostname.rstrip(".").lower()
+    try:
+        ip_address(normalized_host)
+        return None
+    except ValueError:
+        pass
+
+    return await _validate_resolved_host(normalized_host)
+
+
+def _hostname_requiring_dns(url: str) -> str | None:
+    """Return the normalized hostname if DNS validation is needed, None otherwise."""
+    hostname = urlsplit(url).hostname
+    if hostname is None:
+        return None
+    normalized_host = hostname.rstrip(".").lower()
+    try:
+        ip_address(normalized_host)
+        return None
+    except ValueError:
+        return normalized_host
+
+
+def _extract_hostname(url: str) -> str | None:
+    normalized_url = url.strip()
+    if not normalized_url:
+        return None
+
+    hostname = urlsplit(normalized_url).hostname
+    if hostname is None:
+        return None
+
+    return hostname.rstrip(".").lower()
+
+
+async def _validate_resolved_host(host: str) -> tuple[str, str] | None:
+    _, validation_error = await _resolve_and_validate_host(host)
+    return validation_error
+
+
+async def _resolve_and_validate_host(
+    host: str,
+) -> tuple[tuple[str, ...], tuple[str, str] | None]:
+    try:
+        resolved_addresses = await _resolve_host_addresses(host)
+    except socket.gaierror:
+        return (), (
+            "dns",
+            "Target host could not be resolved during safety validation",
+        )
+
+    return resolved_addresses, _block_reason_for_resolved_addresses(resolved_addresses)
+
+
+async def _resolve_host_addresses(host: str) -> tuple[str, ...]:
+    loop = asyncio.get_running_loop()
+    resolved = await loop.run_in_executor(
+        None, lambda: socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)
+    )
+    return tuple(
+        dict.fromkeys(str(ip_address(sockaddr[0])) for _, _, _, _, sockaddr in resolved)
+    )
+
+
+def _block_reason_for_resolved_addresses(
+    resolved_addresses: tuple[str, ...],
+) -> tuple[str, str] | None:
+    if not resolved_addresses:
+        return (
+            "dns",
+            "Target host could not be resolved during safety validation",
+        )
+
+    for resolved_address in resolved_addresses:
+        if not ip_address(resolved_address).is_global:
+            return (
+                "private",
+                "Target host resolves to a private or special-use IP address",
+            )
+
+    return None
+
+
+def _replace_url_host(parsed_url: SplitResult, *, host: str) -> str:
+    netloc = _build_netloc(
+        host=host,
+        port=parsed_url.port,
+        username=parsed_url.username,
+        password=parsed_url.password,
+    )
+    return urlunsplit(
+        (
+            parsed_url.scheme,
+            netloc,
+            parsed_url.path,
+            parsed_url.query,
+            "",
+        )
+    )
+
+
+def _build_netloc(
+    host: str,
+    port: int | None,
+    username: str | None = None,
+    password: str | None = None,
+) -> str:
+    credential_prefix = ""
+    if username is not None:
+        credential_prefix = username
+        if password is not None:
+            credential_prefix = f"{credential_prefix}:{password}"
+        credential_prefix = f"{credential_prefix}@"
+
+    try:
+        host_ip = ip_address(host)
+    except ValueError:
+        normalized_host = host
+    else:
+        normalized_host = (
+            f"[{host_ip.compressed}]" if host_ip.version == 6 else host_ip.compressed
+        )
+
+    if port is None:
+        return f"{credential_prefix}{normalized_host}"
+
+    return f"{credential_prefix}{normalized_host}:{port}"
+
+
+async def resolve_redirect_chain(
+    url: str,
+    *,
+    max_hops: int = _MAX_REDIRECT_HOPS,
+    timeout: float = 10.0,
+) -> str:
+    """Follow HTTP 3xx redirects hop-by-hop, validating each destination
+    against url_safety rules before issuing the next request.
+
+    Returns the final validated URL.
+    Raises CrawlTargetValidationError if any hop is blocked.
+    """
+    current = url
+    async with httpx.AsyncClient(follow_redirects=False, timeout=timeout) as client:
+        for _ in range(max_hops):
+            error = await validate_url(current)
+            if error is not None:
+                category, reason = error
+                raise CrawlTargetValidationError(
+                    [
+                        BlockedCrawlTarget(
+                            hostname=_extract_hostname(current),
+                            category=category,
+                            reason=reason,
+                        )
+                    ]
+                )
+
+            try:
+                resp = await client.head(current)
+            except Exception as exc:
+                _LOGGER.debug(
+                    "Redirect resolution stopped at %s due to network error: %s",
+                    current,
+                    exc,
+                )
+                break
+
+            if resp.status_code not in _REDIRECT_STATUS_CODES:
+                break
+
+            location = resp.headers.get("location")
+            if not location:
+                break
+
+            current = urljoin(current, location)
+
+    error = await validate_url(current)
+    if error is not None:
+        category, reason = error
+        raise CrawlTargetValidationError(
+            [
+                BlockedCrawlTarget(
+                    hostname=_extract_hostname(current),
+                    category=category,
+                    reason=reason,
+                )
+            ]
+        )
+
+    return current

{unique_web_search-2026.22.0.dev0 → unique_web_search-2026.22.0.dev2}/src/unique_web_search/settings.py

@@ -109,6 +109,23 @@ class Base(BaseSettings):
     # LLM processor config override (JSON string matching LLMProcessorConfig schema)
     llm_process_config: str | None = None
 
+    # URL safety settings
+    url_safety_allowed_schemes: list[str] = ["http", "https"]
+    url_safety_localhost_hosts: list[str] = [
+        "localhost",
+        "localhost.localdomain",
+    ]
+    url_safety_metadata_hosts: list[str] = [
+        "100.100.100.200",  # Alibaba Cloud
+        "169.254.169.254",  # AWS / GCP / Azure IMDS
+        "169.254.170.2",  # AWS ECS task credentials
+        "metadata.azure.internal",
+        "metadata.google.internal",
+    ]
+    url_safety_cluster_local_suffix: str = ".cluster.local"
+    url_safety_service_suffix: str = ".svc"
+    url_safety_resolve_redirects: bool = True
+
     @property
     def active_crawlers(self) -> list[str]:
         "Dynamically determine the active crawlers based on the API keys provided"