unique_web_search 2026.24.0.dev0__tar.gz → 2026.24.0.dev2__tar.gz

{unique_web_search-2026.24.0.dev0 → unique_web_search-2026.24.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-web-search
-Version: 2026.24.0.dev0
+Version: 2026.24.0.dev2
 Summary: 
 Author: Andreas Hauri, Gustav Hartz, Rami Azouz
 Author-email: Andreas Hauri <andreas@unique.ch>, Gustav Hartz <gustav.hartz.ext@unique.ch>, Rami Azouz <rami.ext@unique.ch>
@@ -17,7 +17,7 @@ Requires-Dist: tavily-python>=0.7.11,<1
 Requires-Dist: unidecode>=1.4.0,<2
 Requires-Dist: azure-ai-projects>=1.0.0,<2
 Requires-Dist: azure-identity>=1.25.0,<2
-Requires-Dist: unique-toolkit[monitoring]>=2026.24.0.dev0,<2026.24.0rc0
+Requires-Dist: unique-toolkit[monitoring]>=2026.24.0.dev10,<2026.24.0rc0
 Requires-Dist: azure-core>=1.36.0,<2
 Requires-Dist: google-cloud-aiplatform>=1.128.0,<2
 Requires-Dist: google-auth>=2.43.0,<3

{unique_web_search-2026.24.0.dev0 → unique_web_search-2026.24.0.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_web_search"
-version = "2026.24.0.dev0"
+version = "2026.24.0.dev2"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12,<4.0"
@@ -23,7 +23,7 @@ dependencies = [
     "unidecode>=1.4.0,<2",
     "azure-ai-projects>=1.0.0,<2",
     "azure-identity>=1.25.0,<2",
-    "unique-toolkit[monitoring]>=2026.24.0.dev0,<2026.24.0rc0",
+    "unique-toolkit[monitoring]>=2026.24.0.dev10,<2026.24.0rc0",
     "azure-core>=1.36.0,<2",
     "google-cloud-aiplatform>=1.128.0,<2",
     "google-auth>=2.43.0,<3",

{unique_web_search-2026.24.0.dev0 → unique_web_search-2026.24.0.dev2}/src/unique_web_search/prompts.py

@@ -39,11 +39,28 @@ When multiple sources are available, prioritize:
 4. **Primary sources**: Original reports over summaries
 5. **Domain expertise**: Medical info from health institutions, financial from finance sources
 
-**When sources are insufficient**: Clearly state "Based on available search results..." and indicate limitations.
-
 **REMEMBER**: Source numbers reset with each new WebSearch. Only use numbers from the most recent tool response.
 """
 
+_TOOL_FORMAT_INFORMATION_SYNTHESIS_SECTION = """
+## Synthesizing WebSearch results
+
+After one or more WebSearch tool calls, **synthesize** findings in your answer. Do not only cite facts when a single source made the answer obvious.
+
+**Evidence ledger (per facet or sub-question):** Track each facet with a confidence label:
+
+- **Confirmed** — the exact fact is supported by cited tool content [sourceX].
+- **Partial** — only part of the facet is supported; say what is missing [sourceX].
+- **Not found** — no on-topic public evidence after reasonable search/refinement; state what you tried (no citation required for absence).
+- **Related** — tool content is adjacent but does not answer the exact ask; cite it [sourceX] and label it **related, not confirmed**.
+
+**Mandatory behaviors:**
+
+- **Always summarize partial findings** before saying something is unavailable. Never reply with only "I couldn't find X" if the tool returned related material—put related material in a **Related leads** (or equivalent) subsection with citations.
+- For multi-facet tasks (diligence, Source of Wealth, company profiles), use a structured answer: direct answer → evidence by facet → gaps and limitations → related leads → optional next steps.
+- Distinguish **Confirmed** from **Related** in prose—do not imply ownership, revenue, or UBO facts unless citations support them.
+""".strip()
+
 
 _TOOL_FORMAT_INFORMATION_V3_DOMAIN_DIVERSITY_SECTION = """
 ## Domain Diversity Requirement
@@ -61,4 +78,6 @@ DEFAULT_TOOL_FORMAT_INFORMATION_FOR_SYSTEM_PROMPT_V3 = (
     DEFAULT_TOOL_FORMAT_INFORMATION_FOR_SYSTEM_PROMPT.rstrip()
     + "\n\n"
     + _TOOL_FORMAT_INFORMATION_V3_DOMAIN_DIVERSITY_SECTION
+    + "\n\n"
+    + _TOOL_FORMAT_INFORMATION_SYNTHESIS_SECTION
 )

{unique_web_search-2026.24.0.dev0 → unique_web_search-2026.24.0.dev2}/src/unique_web_search/services/crawlers/README.md

@@ -4,6 +4,17 @@ This directory contains implementations of the different web crawlers used by `u
 
 All crawlers inherit from the shared base classes in `base.py`.
 
+### URL safety
+
+Crawlers route every URL through `services/crawlers/url_safety/` before fetching:
+
+- **`BaseCrawler.crawl`** calls `UrlSafetyService.validate_batch_urls` first, then passes `list[ResolvedCrawlTarget]` to each crawler’s `_crawl`. When `url_safety_settings.enabled` is `False`, validation is bypassed and each URL is wrapped via `bypass_crawl_target` without DNS resolution or policy checks.
+- **`BasicCrawler`** uses `ResolvedCrawlTarget.request_url`, `host_header`, and `sni_hostname` for DNS pinning (single resolve inside `validate_batch_urls`).
+- **`Crawl4AICrawler`** registers a Playwright `before_goto` hook that calls `UrlSafetyService.validate_url` on every navigation when `url_safety_settings.enabled` is `True`.
+- **API crawlers** (Tavily, Firecrawl, Jina) use `target.normalized_url` from the validated targets.
+
+Settings live in `url_safety/settings.py` (`url_safety_settings`, env prefix `URL_SAFETY_`). Key fields include `enabled` (`URL_SAFETY_ENABLED`), `resolve_redirects` (`URL_SAFETY_RESOLVE_REDIRECTS`), and the host/scheme allowlists (`allowed_schemes`, `localhost_hosts`, `metadata_hosts`, `cluster_local_suffix`, `service_suffix`).
+
 ```python
 class BaseCrawlerConfig(BaseModel):
     crawler_type: CrawlerType

{unique_web_search-2026.24.0.dev0 → unique_web_search-2026.24.0.dev2}/src/unique_web_search/services/crawlers/base.py

@@ -1,4 +1,3 @@
-import asyncio
 from abc import ABC, abstractmethod
 from enum import StrEnum
 from typing import Generic, TypeVar
@@ -8,15 +7,14 @@ from unique_toolkit.agentic.tools.config import (
     get_configuration_dict,
 )
 
+from unique_web_search.services.crawlers.url_safety import (
+    ResolvedCrawlTarget,
+    UrlSafetyService,
+)
 from unique_web_search.services.helpers import (
     clean_model_title_generator,
     experimental_model_title_generator,
 )
-from unique_web_search.services.url_safety import (
-    resolve_redirect_chain,
-    validate_crawl_urls,
-)
-from unique_web_search.settings import env_settings
 
 
 class CrawlerType(StrEnum):
@@ -57,12 +55,8 @@ class BaseCrawler(ABC, Generic[CrawlerConfig]):
         self.config = config
 
     async def crawl(self, urls: list[str]) -> list[str]:
-        urls = await validate_crawl_urls(urls)
-        if env_settings.url_safety_resolve_redirects:
-            urls = list(
-                await asyncio.gather(*[resolve_redirect_chain(u) for u in urls])
-            )
-        return await self._crawl(urls)
+        targets = await UrlSafetyService.validate_batch_urls(urls)
+        return await self._crawl(targets)
 
     @abstractmethod
-    async def _crawl(self, urls: list[str]) -> list[str]: ...
+    async def _crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]: ...

{unique_web_search-2026.24.0.dev0 → unique_web_search-2026.24.0.dev2}/src/unique_web_search/services/crawlers/basic.py

@@ -15,11 +15,11 @@ from unique_web_search.services.crawlers.base import (
     BaseCrawlerConfig,
     CrawlerType,
 )
-from unique_web_search.services.crawlers.utils import get_random_user_agent
-from unique_web_search.services.url_safety import (
+from unique_web_search.services.crawlers.url_safety import (
     CrawlTargetValidationError,
-    resolve_crawl_target,
+    ResolvedCrawlTarget,
 )
+from unique_web_search.services.crawlers.utils import get_random_user_agent
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -63,9 +63,9 @@ class BasicCrawler(BaseCrawler[BasicCrawlerConfig]):
     #     tags=["basic", "scrape"],
     # )
     @override
-    async def _crawl(self, urls: list[str]) -> list[str]:
+    async def _crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]:
         async with async_client(timeout=Timeout(self.config.timeout)) as client:
-            tasks = [self._crawl_url_with_client(client, url) for url in urls]
+            tasks = [self._crawl_url_with_client(client, target) for target in targets]
             results = await asyncio.gather(*tasks, return_exceptions=True)
 
             validation_errors = [
@@ -88,19 +88,23 @@ class BasicCrawler(BaseCrawler[BasicCrawlerConfig]):
 
             return markdowns
 
-    async def _crawl_url_with_client(self, client: AsyncClient, url: str) -> str:
+    async def _crawl_url_with_client(
+        self, client: AsyncClient, target: ResolvedCrawlTarget
+    ) -> str:
         headers = {"User-Agent": get_random_user_agent()}
 
-        if self._is_url_blacklisted(url):
+        if self._is_url_blacklisted(target.normalized_url):
             return "Unable to crawl URL due to blacklisted pattern"
 
-        (
-            request_url,
-            request_headers,
-            request_extensions,
-        ) = await self._build_request_target(url, headers)
+        request_headers = dict(headers)
+        request_extensions: dict[str, str] = {}
+        if target.host_header is not None:
+            request_headers["Host"] = target.host_header
+        if target.sni_hostname is not None:
+            request_extensions["sni_hostname"] = target.sni_hostname
+
         response = await client.get(
-            request_url,
+            target.request_url,
             headers=request_headers,
             extensions=request_extensions or None,
         )
@@ -128,22 +132,6 @@ class BasicCrawler(BaseCrawler[BasicCrawlerConfig]):
     def _content_type_not_allowed(self, content_type: str) -> bool:
         return content_type in self.config.unwanted_content_types
 
-    async def _build_request_target(
-        self,
-        url: str,
-        headers: dict[str, str],
-    ) -> tuple[str, dict[str, str], dict[str, str]]:
-        resolved_target = await resolve_crawl_target(url)
-
-        request_headers = dict(headers)
-        request_extensions: dict[str, str] = {}
-        if resolved_target.host_header is not None:
-            request_headers["Host"] = resolved_target.host_header
-        if resolved_target.sni_hostname is not None:
-            request_extensions["sni_hostname"] = resolved_target.sni_hostname
-
-        return resolved_target.request_url, request_headers, request_extensions
-
 
 def _markdownify_html_with_timeout(content: str, timeout: float) -> str:
     @timeout_decorator.timeout(timeout)

{unique_web_search-2026.24.0.dev0 → unique_web_search-2026.24.0.dev2}/src/unique_web_search/services/crawlers/crawl4ai.py

@@ -10,8 +10,12 @@ from unique_web_search.services.crawlers.base import (
     BaseCrawlerConfig,
     CrawlerType,
 )
+from unique_web_search.services.crawlers.url_safety import (
+    ResolvedCrawlTarget,
+    UrlSafetyService,
+    url_safety_settings,
+)
 from unique_web_search.services.crawlers.utils import get_random_user_agent
-from unique_web_search.services.url_safety import validate_url
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -188,7 +192,7 @@ class Crawl4AiCrawler(BaseCrawler[Crawl4AiCrawlerConfig]):
 
             async def _route_handler(route: Route, request: Request) -> None:
                 req_url = request.url
-                error = await validate_url(req_url)
+                error = await UrlSafetyService.validate_url(req_url)
                 if error is not None:
                     category, reason = error
                     _LOGGER.warning(
@@ -205,7 +209,8 @@ class Crawl4AiCrawler(BaseCrawler[Crawl4AiCrawlerConfig]):
 
         return _ssrf_guard_hook
 
-    async def _crawl(self, urls: list[str]) -> list[str]:
+    async def _crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]:
+        urls = [target.normalized_url for target in targets]
         # Lazy import of crawl4ai - only import when actually needed
         from crawl4ai import (
             AsyncWebCrawler,
@@ -276,11 +281,14 @@ class Crawl4AiCrawler(BaseCrawler[Crawl4AiCrawlerConfig]):
             rate_limiter=rate_limiter,
         )
 
+        hooks = (
+            {"before_goto": self._get_ssrf_guard_hook()}
+            if url_safety_settings.enabled
+            else {}
+        )
         crawler_strategy = AsyncPlaywrightCrawlerStrategy(
             browser_config=browser_config,
-            hooks={
-                "before_goto": self._get_ssrf_guard_hook(),
-            },
+            hooks=hooks,
         )
 
         _LOGGER.info(f"Crawling {len(urls)} URLs with Crawl4AiCrawler")

{unique_web_search-2026.24.0.dev0 → unique_web_search-2026.24.0.dev2}/src/unique_web_search/services/crawlers/firecrawl.py

@@ -10,6 +10,7 @@ from unique_web_search.services.crawlers.base import (
     BaseCrawlerConfigExperimental,
     CrawlerType,
 )
+from unique_web_search.services.crawlers.url_safety import ResolvedCrawlTarget
 
 
 class FirecrawlCrawlerConfig(BaseCrawlerConfigExperimental[CrawlerType.FIRECRAWL]):
@@ -24,7 +25,8 @@ class FirecrawlCrawler(BaseCrawler[FirecrawlCrawlerConfig]):
     # @track(
     #     tags=["firecrawl", "scrape"],
     # )
-    async def _crawl(self, urls: list[str]) -> list[str]:
+    async def _crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]:
+        urls = [target.normalized_url for target in targets]
         api_key = get_firecrawl_search_settings().api_key
         assert api_key is not None, "Firecrawl API key is not configured"
 

{unique_web_search-2026.24.0.dev0 → unique_web_search-2026.24.0.dev2}/src/unique_web_search/services/crawlers/jina.py

@@ -10,6 +10,7 @@ from unique_web_search.services.crawlers.base import (
     BaseCrawlerConfigExperimental,
     CrawlerType,
 )
+from unique_web_search.services.crawlers.url_safety import ResolvedCrawlTarget
 
 
 class ReaderBody(BaseModel):
@@ -52,7 +53,8 @@ class JinaCrawler(BaseCrawler[JinaCrawlerConfig]):
     # @track(
     #     tags=["jina", "scrape"],
     # )
-    async def _crawl(self, urls: list[str]) -> list[str]:
+    async def _crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]:
+        urls = [target.normalized_url for target in targets]
         jina_settings = get_jina_search_settings()
         api_key = jina_settings.api_key
         assert api_key is not None, "Jina API key is not configured"

{unique_web_search-2026.24.0.dev0 → unique_web_search-2026.24.0.dev2}/src/unique_web_search/services/crawlers/tavily.py

@@ -9,6 +9,7 @@ from unique_web_search.services.crawlers.base import (
     BaseCrawlerConfigExperimental,
     CrawlerType,
 )
+from unique_web_search.services.crawlers.url_safety import ResolvedCrawlTarget
 
 
 class TavilyCrawlerConfig(BaseCrawlerConfigExperimental[CrawlerType.TAVILY]):
@@ -24,7 +25,8 @@ class TavilyCrawler(BaseCrawler[TavilyCrawlerConfig]):
     # @track(
     #     tags=["tavily", "scrape"],
     # )
-    async def _crawl(self, urls: list[str]) -> list[str]:
+    async def _crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]:
+        urls = [target.normalized_url for target in targets]
         api_key = get_tavily_search_settings().api_key
         assert api_key is not None, "Tavily API key is not configured"
 

unique_web_search-2026.24.0.dev2/src/unique_web_search/services/crawlers/url_safety/__init__.py

@@ -0,0 +1,15 @@
+from unique_web_search.services.crawlers.url_safety.models import (
+    BlockedCrawlTarget,
+    CrawlTargetValidationError,
+    ResolvedCrawlTarget,
+)
+from unique_web_search.services.crawlers.url_safety.service import UrlSafetyService
+from unique_web_search.services.crawlers.url_safety.settings import url_safety_settings
+
+__all__ = [
+    "BlockedCrawlTarget",
+    "url_safety_settings",
+    "CrawlTargetValidationError",
+    "ResolvedCrawlTarget",
+    "UrlSafetyService",
+]

unique_web_search-2026.24.0.dev2/src/unique_web_search/services/crawlers/url_safety/dns.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import asyncio
+import socket
+from ipaddress import ip_address
+
+
+async def resolve_host_addresses(host: str) -> tuple[str, ...]:
+    loop = asyncio.get_running_loop()
+    resolved = await loop.run_in_executor(
+        None, lambda: socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)
+    )
+    return tuple(
+        dict.fromkeys(str(ip_address(sockaddr[0])) for _, _, _, _, sockaddr in resolved)
+    )
+
+
+def block_reason_for_resolved_addresses(
+    resolved_addresses: tuple[str, ...],
+) -> tuple[str, str] | None:
+    if not resolved_addresses:
+        return (
+            "dns",
+            "Target host could not be resolved during safety validation",
+        )
+
+    for resolved_address in resolved_addresses:
+        if not ip_address(resolved_address).is_global:
+            return (
+                "private",
+                "Target host resolves to a private or special-use IP address",
+            )
+
+    return None
+
+
+async def resolve_and_validate_host(
+    host: str,
+) -> tuple[tuple[str, ...], tuple[str, str] | None]:
+    try:
+        resolved_addresses = await resolve_host_addresses(host)
+    except socket.gaierror:
+        return (), (
+            "dns",
+            "Target host could not be resolved during safety validation",
+        )
+
+    return resolved_addresses, block_reason_for_resolved_addresses(resolved_addresses)
+
+
+async def validate_resolved_host(host: str) -> tuple[str, str] | None:
+    _, validation_error = await resolve_and_validate_host(host)
+    return validation_error

unique_web_search-2026.24.0.dev2/src/unique_web_search/services/crawlers/url_safety/models.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from urllib.parse import urlsplit
+
+from unique_web_search.metrics import crawl_blocked
+from unique_web_search.services.crawlers.url_safety.netloc import (
+    build_netloc,
+    replace_url_host,
+)
+
+
+@dataclass(frozen=True)
+class BlockedCrawlTarget:
+    hostname: str | None
+    category: str
+    reason: str
+
+    @property
+    def display_target(self) -> str:
+        return self.hostname or "<unknown-host>"
+
+
+@dataclass(frozen=True)
+class ResolvedCrawlTarget:
+    """Carries the validated host/IP values reused when issuing crawl requests."""
+
+    normalized_url: str
+    hostname: str
+    resolved_ip: str
+    used_dns_resolution: bool
+
+    @property
+    def request_url(self) -> str:
+        if not self.used_dns_resolution and not self.resolved_ip:
+            return self.normalized_url
+        return replace_url_host(urlsplit(self.normalized_url), host=self.resolved_ip)
+
+    @property
+    def host_header(self) -> str | None:
+        if not self.used_dns_resolution:
+            return None
+
+        return build_netloc(host=self.hostname, port=urlsplit(self.normalized_url).port)
+
+    @property
+    def sni_hostname(self) -> str | None:
+        if not self.used_dns_resolution:
+            return None
+
+        if urlsplit(self.normalized_url).scheme.lower() != "https":
+            return None
+
+        return self.hostname
+
+
+class CrawlTargetValidationError(ValueError):
+    def __init__(self, blocked_targets: list[BlockedCrawlTarget]):
+        self.blocked_targets: list[BlockedCrawlTarget] = blocked_targets
+
+        for target in blocked_targets:
+            crawl_blocked.labels(reason_category=target.category).inc()
+
+        details = "; ".join(
+            f"{target.display_target} ({target.reason})"
+            for target in self.blocked_targets
+        )
+        super().__init__(f"Blocked crawl target(s) due to URL safety policy: {details}")
+
+
+def bypass_crawl_target(url: str) -> ResolvedCrawlTarget:
+    return ResolvedCrawlTarget(
+        normalized_url=url.strip(),
+        hostname="",
+        resolved_ip="",
+        used_dns_resolution=False,
+    )

unique_web_search-2026.24.0.dev2/src/unique_web_search/services/crawlers/url_safety/netloc.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from ipaddress import ip_address
+from urllib.parse import SplitResult, urlsplit, urlunsplit
+
+
+def extract_hostname(url: str) -> str | None:
+    normalized_url = url.strip()
+    if not normalized_url:
+        return None
+
+    hostname = urlsplit(normalized_url).hostname
+    if hostname is None:
+        return None
+
+    return hostname.rstrip(".").lower()
+
+
+def replace_url_host(parsed_url: SplitResult, *, host: str) -> str:
+    netloc = build_netloc(
+        host=host,
+        port=parsed_url.port,
+        username=parsed_url.username,
+        password=parsed_url.password,
+    )
+    return urlunsplit(
+        (
+            parsed_url.scheme,
+            netloc,
+            parsed_url.path,
+            parsed_url.query,
+            "",
+        )
+    )
+
+
+def build_netloc(
+    host: str,
+    port: int | None,
+    username: str | None = None,
+    password: str | None = None,
+) -> str:
+    credential_prefix = ""
+    if username is not None:
+        credential_prefix = username
+        if password is not None:
+            credential_prefix = f"{credential_prefix}:{password}"
+        credential_prefix = f"{credential_prefix}@"
+
+    try:
+        host_ip = ip_address(host)
+    except ValueError:
+        normalized_host = host
+    else:
+        normalized_host = (
+            f"[{host_ip.compressed}]" if host_ip.version == 6 else host_ip.compressed
+        )
+
+    if port is None:
+        return f"{credential_prefix}{normalized_host}"
+
+    return f"{credential_prefix}{normalized_host}:{port}"

unique_web_search-2026.24.0.dev2/src/unique_web_search/services/crawlers/url_safety/policy.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from ipaddress import ip_address
+from urllib.parse import urlsplit
+
+from unique_web_search.services.crawlers.url_safety.settings import url_safety_settings
+
+
+def validate_target_cheap(url: str) -> tuple[str, str] | None:
+    """Static validation only — no DNS. Returns (category, reason) if blocked, None otherwise."""
+    if not url:
+        return "empty", "URL is empty"
+
+    parsed_url = urlsplit(url)
+    if parsed_url.scheme.lower() not in url_safety_settings.allowed_schemes:
+        return "scheme", "URL scheme must be http or https"
+
+    hostname = parsed_url.hostname
+    if not hostname:
+        return "host", "URL host is missing or malformed"
+
+    normalized_host = hostname.rstrip(".").lower()
+
+    if normalized_host in url_safety_settings.metadata_hosts:
+        return "metadata", "Target points to a metadata endpoint"
+
+    if (
+        normalized_host in url_safety_settings.localhost_hosts
+        or normalized_host.endswith(".localhost")
+    ):
+        return "localhost", "Target points to a localhost host"
+
+    if normalized_host.endswith(url_safety_settings.service_suffix):
+        return "cluster", "Target points to an internal service host"
+
+    if normalized_host.endswith(url_safety_settings.cluster_local_suffix):
+        return "cluster", "Target points to an internal cluster-local host"
+
+    try:
+        target_ip = ip_address(normalized_host)
+    except ValueError:
+        pass
+    else:
+        if not target_ip.is_global:
+            return "private", "Target points to a private or special-use IP address"
+
+        return None
+
+    if "." not in normalized_host:
+        return "cluster", "Target points to a single-label internal host"
+
+    return None

unique_web_search-2026.24.0.dev2/src/unique_web_search/services/crawlers/url_safety/redirect.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Awaitable, Callable
+from urllib.parse import urljoin
+
+import httpx
+
+from unique_web_search.services.crawlers.url_safety.models import (
+    BlockedCrawlTarget,
+    CrawlTargetValidationError,
+)
+from unique_web_search.services.crawlers.url_safety.netloc import extract_hostname
+from unique_web_search.services.crawlers.url_safety.settings import url_safety_settings
+
+_LOGGER = logging.getLogger(__name__)
+
+_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})
+
+ValidateUrlFn = Callable[[str], Awaitable[tuple[str, str] | None]]
+
+
+async def resolve_redirect_chain(
+    url: str,
+    validate_url: ValidateUrlFn,
+) -> str:
+    """Follow HTTP 3xx redirects hop-by-hop, validating each destination.
+
+    Returns the final validated URL.
+    Raises CrawlTargetValidationError if any hop is blocked.
+    """
+    current = url
+    timeout = url_safety_settings.redirect_timeout_seconds
+    async with httpx.AsyncClient(follow_redirects=False, timeout=timeout) as client:
+        for _ in range(url_safety_settings.max_redirect_hops):
+            error = await validate_url(current)
+            if error is not None:
+                category, reason = error
+                raise CrawlTargetValidationError(
+                    [
+                        BlockedCrawlTarget(
+                            hostname=extract_hostname(current),
+                            category=category,
+                            reason=reason,
+                        )
+                    ]
+                )
+
+            try:
+                resp = await client.head(current)
+            except Exception as exc:
+                _LOGGER.debug(
+                    "Redirect resolution stopped at %s due to network error: %s",
+                    current,
+                    exc,
+                )
+                break
+
+            if resp.status_code not in _REDIRECT_STATUS_CODES:
+                break
+
+            location = resp.headers.get("location")
+            if not location:
+                break
+
+            current = urljoin(current, location)
+
+    error = await validate_url(current)
+    if error is not None:
+        category, reason = error
+        raise CrawlTargetValidationError(
+            [
+                BlockedCrawlTarget(
+                    hostname=extract_hostname(current),
+                    category=category,
+                    reason=reason,
+                )
+            ]
+        )
+
+    return current