unique-web-search 2026.26.0.dev7__tar.gz → 2026.26.0.dev8__tar.gz

{unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-web-search
-Version: 2026.26.0.dev7
+Version: 2026.26.0.dev8
 Summary: 
 Author: Andreas Hauri, Gustav Hartz, Rami Azouz
 Author-email: Andreas Hauri <andreas@unique.ch>, Gustav Hartz <gustav.hartz.ext@unique.ch>, Rami Azouz <rami.ext@unique.ch>
@@ -28,8 +28,7 @@ Requires-Dist: certifi>=2025.11.12
 Requires-Dist: jinja2>=3.1.6
 Requires-Dist: playwright>=1.58.0
 Requires-Dist: perplexityai>=0.34.0
-Requires-Dist: unique-search-proxy-core>=2026.26.0.dev5,<2026.26.0rc0
-Requires-Dist: unique-search-proxy-sdk>=2026.26.0.dev5,<2026.26.0rc0
+Requires-Dist: unique-search-proxy-core>=2026.26.0.dev6,<2026.26.0rc0
 Requires-Python: >=3.12, <4.0
 Description-Content-Type: text/markdown
 

{unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_web_search"
-version = "2026.26.0.dev7"
+version = "2026.26.0.dev8"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12,<4.0"
@@ -34,8 +34,8 @@ dependencies = [
     "jinja2>=3.1.6",
     "playwright>=1.58.0",
     "perplexityai>=0.34.0",
-    "unique-search-proxy-core>=2026.26.0.dev5,<2026.26.0rc0",
-    "unique-search-proxy-sdk>=2026.26.0.dev5,<2026.26.0rc0",
+    "unique-search-proxy-core>=2026.26.0.dev6,<2026.26.0rc0",
+    # "unique-search-proxy-sdk>=2026.24.0.dev0",
 ]
 
 [build-system]
@@ -93,5 +93,5 @@ extend-select = ["I"]
 
 [tool.uv.sources]
 unique-toolkit = { workspace = true }
-unique-search-proxy-sdk = { workspace = true }
+# unique-search-proxy-sdk = { workspace = true }
 unique-search-proxy-core = { workspace = true }

{unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev8}/src/unique_web_search/services/crawlers/__init__.py

@@ -9,10 +9,6 @@ from unique_web_search.services.crawlers.basic import (
     BasicCrawler,
     BasicCrawlerConfig,
 )
-from unique_web_search.services.crawlers.basic_proxy import (
-    BasicCrawlerProxyConfig,
-    BasicProxyCrawler,
-)
 from unique_web_search.services.crawlers.crawl4ai import (
     Crawl4AiCrawler,
     Crawl4AiCrawlerConfig,
@@ -31,17 +27,11 @@ from unique_web_search.services.crawlers.tavily import (
 )
 
 CrawlerTypes = (
-    BasicCrawler
-    | BasicProxyCrawler
-    | Crawl4AiCrawler
-    | FirecrawlCrawler
-    | JinaCrawler
-    | TavilyCrawler
+    BasicCrawler | Crawl4AiCrawler | FirecrawlCrawler | JinaCrawler | TavilyCrawler
 )
 
 CrawlerConfigTypes = (
     BasicCrawlerConfig
-    | BasicCrawlerProxyConfig
     | Crawl4AiCrawlerConfig
     | FirecrawlCrawlerConfig
     | JinaCrawlerConfig
@@ -54,7 +44,6 @@ CRAWLER_NAME_TO_CONFIG = {
     "firecrawl": FirecrawlCrawlerConfig,
     "jina": JinaCrawlerConfig,
     "tavily": TavilyCrawlerConfig,
-    "basic_proxy": BasicCrawlerProxyConfig,
 }
 
 
@@ -72,8 +61,6 @@ def get_crawler_service(
             return FirecrawlCrawler(crawler_config)
         case CrawlerType.JINA:
             return JinaCrawler(crawler_config)
-        case CrawlerType.BASIC_PROXY:
-            return BasicProxyCrawler(crawler_config)
 
 
 def get_crawler_config_types_from_names(crawler_names: list[str]) -> TypeAlias:
@@ -103,8 +90,6 @@ def get_default_crawler_config(
 __all__ = [
     "BasicCrawler",
     "BasicCrawlerConfig",
-    "BasicProxyCrawler",
-    "BasicCrawlerProxyConfig",
     "Crawl4AiCrawler",
     "Crawl4AiCrawlerConfig",
     "FirecrawlCrawler",

{unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev8}/src/unique_web_search/services/crawlers/base.py

@@ -3,14 +3,16 @@ from enum import StrEnum
 from typing import Generic, TypeVar
 
 from pydantic import BaseModel
+from unique_search_proxy_core.url_safety import (
+    CrawlTargetValidationError,
+    ResolvedCrawlTarget,
+    UrlSafetyService,
+)
 from unique_toolkit.agentic.tools.config import (
     get_configuration_dict,
 )
 
-from unique_web_search.services.crawlers.url_safety import (
-    ResolvedCrawlTarget,
-    UrlSafetyService,
-)
+from unique_web_search.metrics import crawl_blocked
 from unique_web_search.services.helpers import (
     clean_model_title_generator,
     experimental_model_title_generator,
@@ -20,7 +22,6 @@ from unique_web_search.services.helpers import (
 class CrawlerType(StrEnum):
     CRAWL4AI = "Crawl4AiCrawler"
     BASIC = "BasicCrawler"
-    BASIC_PROXY = "BasicProxyCrawler"
     NO_CRAWLER = "NoCrawler"
     TAVILY = "TavilyCrawler"
     FIRECRAWL = "FirecrawlCrawler"
@@ -55,7 +56,12 @@ class BaseCrawler(ABC, Generic[CrawlerConfig]):
         self.config = config
 
     async def crawl(self, urls: list[str]) -> list[str]:
-        targets = await UrlSafetyService.validate_batch_urls(urls)
+        try:
+            targets = await UrlSafetyService.validate_batch_urls(urls)
+        except CrawlTargetValidationError as exc:
+            for target in exc.blocked_targets:
+                crawl_blocked.labels(reason_category=target.category).inc()
+            raise
         return await self._crawl(targets)
 
     @abstractmethod

unique_web_search-2026.26.0.dev8/src/unique_web_search/services/crawlers/url_safety/__init__.py

@@ -0,0 +1,21 @@
+"""Deprecated re-exports — use ``unique_search_proxy_core.url_safety`` directly."""
+
+from unique_search_proxy_core.url_safety import (
+    BlockedCrawlTarget,
+    CrawlTargetValidationError,
+    ResolvedCrawlTarget,
+    UrlSafetyOutcome,
+    UrlSafetyService,
+    UrlSafetySettings,
+    url_safety_settings,
+)
+
+__all__ = [
+    "BlockedCrawlTarget",
+    "CrawlTargetValidationError",
+    "ResolvedCrawlTarget",
+    "UrlSafetyOutcome",
+    "UrlSafetyService",
+    "UrlSafetySettings",
+    "url_safety_settings",
+]

unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/basic_proxy.py

@@ -1,79 +0,0 @@
-import logging
-from typing import Literal
-
-from unique_search_proxy_core.crawlers import (
-    BasicConfig as BasicCrawlerProxyCoreConfig,
-)
-from unique_search_proxy_core.crawlers.base import CrawlerType as ProxyCrawlerType
-from unique_search_proxy_sdk import UniqueSearchProxyClient
-from unique_search_proxy_sdk._generated.models.crawl_url_result import CrawlUrlResult
-from unique_search_proxy_sdk._generated.models.per_url_error import PerUrlError
-
-from unique_web_search.client_settings import get_search_proxy_settings
-from unique_web_search.services.crawlers.base import BaseCrawlerConfig, CrawlerType
-
-_LOGGER = logging.getLogger(__name__)
-
-
-class BasicCrawlerProxyConfig(
-    BaseCrawlerConfig[CrawlerType.BASIC_PROXY], BasicCrawlerProxyCoreConfig
-):
-    crawler_type: Literal[CrawlerType.BASIC_PROXY] = CrawlerType.BASIC_PROXY
-
-
-class BasicProxyCrawler:
-    """Crawler that delegates fetching to the remote Unique Search Proxy service.
-
-    Intermediate transition crawler: instead of fetching URLs in-process (see
-    :class:`BasicCrawler`), it forwards the request to the search-proxy service
-    via :class:`UniqueSearchProxyClient` and maps the per-URL results back to the
-    markdown list contract shared by the other crawlers.
-    """
-
-    def __init__(self, config: BasicCrawlerProxyConfig):
-        self.config: BasicCrawlerProxyConfig = config
-
-    async def crawl(self, urls: list[str]) -> list[str]:
-        return await self._crawl(urls)
-
-    async def _crawl(self, urls: list[str]) -> list[str]:
-        base_url = get_search_proxy_settings().base_url
-        assert base_url is not None, "Unique Search Proxy base URL is not configured"
-
-        try:
-            async with UniqueSearchProxyClient(
-                base_url=base_url,
-                timeout=float(self.config.timeout),
-            ) as client:
-                response = await client.crawl.crawl(
-                    urls,
-                    crawler=ProxyCrawlerType.BASIC,
-                    content_types=self.config.content_types,
-                    timeout=self.config.timeout,
-                    max_concurrent_requests=self.config.max_concurrent_requests,
-                )
-        except Exception as error:
-            _LOGGER.exception("Unique Search Proxy crawl request failed")
-            return [f"Unable to crawl URL via search proxy: {error}" for _ in urls]
-
-        url_to_markdown = {
-            result.url: _result_to_markdown(result) for result in response.results
-        }
-
-        return [
-            url_to_markdown.get(url, "Error: URL not found in search proxy response")
-            for url in urls
-        ]
-
-
-def _result_to_markdown(result: CrawlUrlResult) -> str:
-    if isinstance(result.content, str):
-        return result.content
-
-    if isinstance(result.error, PerUrlError):
-        return f"Error: {result.error.message}"
-
-    if isinstance(result.raw, str):
-        return result.raw
-
-    return "Error: No content returned by search proxy"

unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/__init__.py

@@ -1,15 +0,0 @@
-from unique_web_search.services.crawlers.url_safety.models import (
-    BlockedCrawlTarget,
-    CrawlTargetValidationError,
-    ResolvedCrawlTarget,
-)
-from unique_web_search.services.crawlers.url_safety.service import UrlSafetyService
-from unique_web_search.services.crawlers.url_safety.settings import url_safety_settings
-
-__all__ = [
-    "BlockedCrawlTarget",
-    "url_safety_settings",
-    "CrawlTargetValidationError",
-    "ResolvedCrawlTarget",
-    "UrlSafetyService",
-]

unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/dns.py

@@ -1,53 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import socket
-from ipaddress import ip_address
-
-
-async def resolve_host_addresses(host: str) -> tuple[str, ...]:
-    loop = asyncio.get_running_loop()
-    resolved = await loop.run_in_executor(
-        None, lambda: socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)
-    )
-    return tuple(
-        dict.fromkeys(str(ip_address(sockaddr[0])) for _, _, _, _, sockaddr in resolved)
-    )
-
-
-def block_reason_for_resolved_addresses(
-    resolved_addresses: tuple[str, ...],
-) -> tuple[str, str] | None:
-    if not resolved_addresses:
-        return (
-            "dns",
-            "Target host could not be resolved during safety validation",
-        )
-
-    for resolved_address in resolved_addresses:
-        if not ip_address(resolved_address).is_global:
-            return (
-                "private",
-                "Target host resolves to a private or special-use IP address",
-            )
-
-    return None
-
-
-async def resolve_and_validate_host(
-    host: str,
-) -> tuple[tuple[str, ...], tuple[str, str] | None]:
-    try:
-        resolved_addresses = await resolve_host_addresses(host)
-    except socket.gaierror:
-        return (), (
-            "dns",
-            "Target host could not be resolved during safety validation",
-        )
-
-    return resolved_addresses, block_reason_for_resolved_addresses(resolved_addresses)
-
-
-async def validate_resolved_host(host: str) -> tuple[str, str] | None:
-    _, validation_error = await resolve_and_validate_host(host)
-    return validation_error

unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/models.py

@@ -1,77 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from urllib.parse import urlsplit
-
-from unique_web_search.metrics import crawl_blocked
-from unique_web_search.services.crawlers.url_safety.netloc import (
-    build_netloc,
-    replace_url_host,
-)
-
-
-@dataclass(frozen=True)
-class BlockedCrawlTarget:
-    hostname: str | None
-    category: str
-    reason: str
-
-    @property
-    def display_target(self) -> str:
-        return self.hostname or "<unknown-host>"
-
-
-@dataclass(frozen=True)
-class ResolvedCrawlTarget:
-    """Carries the validated host/IP values reused when issuing crawl requests."""
-
-    normalized_url: str
-    hostname: str
-    resolved_ip: str
-    used_dns_resolution: bool
-
-    @property
-    def request_url(self) -> str:
-        if not self.used_dns_resolution and not self.resolved_ip:
-            return self.normalized_url
-        return replace_url_host(urlsplit(self.normalized_url), host=self.resolved_ip)
-
-    @property
-    def host_header(self) -> str | None:
-        if not self.used_dns_resolution:
-            return None
-
-        return build_netloc(host=self.hostname, port=urlsplit(self.normalized_url).port)
-
-    @property
-    def sni_hostname(self) -> str | None:
-        if not self.used_dns_resolution:
-            return None
-
-        if urlsplit(self.normalized_url).scheme.lower() != "https":
-            return None
-
-        return self.hostname
-
-
-class CrawlTargetValidationError(ValueError):
-    def __init__(self, blocked_targets: list[BlockedCrawlTarget]):
-        self.blocked_targets: list[BlockedCrawlTarget] = blocked_targets
-
-        for target in blocked_targets:
-            crawl_blocked.labels(reason_category=target.category).inc()
-
-        details = "; ".join(
-            f"{target.display_target} ({target.reason})"
-            for target in self.blocked_targets
-        )
-        super().__init__(f"Blocked crawl target(s) due to URL safety policy: {details}")
-
-
-def bypass_crawl_target(url: str) -> ResolvedCrawlTarget:
-    return ResolvedCrawlTarget(
-        normalized_url=url.strip(),
-        hostname="",
-        resolved_ip="",
-        used_dns_resolution=False,
-    )

unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/netloc.py

@@ -1,62 +0,0 @@
-from __future__ import annotations
-
-from ipaddress import ip_address
-from urllib.parse import SplitResult, urlsplit, urlunsplit
-
-
-def extract_hostname(url: str) -> str | None:
-    normalized_url = url.strip()
-    if not normalized_url:
-        return None
-
-    hostname = urlsplit(normalized_url).hostname
-    if hostname is None:
-        return None
-
-    return hostname.rstrip(".").lower()
-
-
-def replace_url_host(parsed_url: SplitResult, *, host: str) -> str:
-    netloc = build_netloc(
-        host=host,
-        port=parsed_url.port,
-        username=parsed_url.username,
-        password=parsed_url.password,
-    )
-    return urlunsplit(
-        (
-            parsed_url.scheme,
-            netloc,
-            parsed_url.path,
-            parsed_url.query,
-            "",
-        )
-    )
-
-
-def build_netloc(
-    host: str,
-    port: int | None,
-    username: str | None = None,
-    password: str | None = None,
-) -> str:
-    credential_prefix = ""
-    if username is not None:
-        credential_prefix = username
-        if password is not None:
-            credential_prefix = f"{credential_prefix}:{password}"
-        credential_prefix = f"{credential_prefix}@"
-
-    try:
-        host_ip = ip_address(host)
-    except ValueError:
-        normalized_host = host
-    else:
-        normalized_host = (
-            f"[{host_ip.compressed}]" if host_ip.version == 6 else host_ip.compressed
-        )
-
-    if port is None:
-        return f"{credential_prefix}{normalized_host}"
-
-    return f"{credential_prefix}{normalized_host}:{port}"

unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/policy.py

@@ -1,52 +0,0 @@
-from __future__ import annotations
-
-from ipaddress import ip_address
-from urllib.parse import urlsplit
-
-from unique_web_search.services.crawlers.url_safety.settings import url_safety_settings
-
-
-def validate_target_cheap(url: str) -> tuple[str, str] | None:
-    """Static validation only — no DNS. Returns (category, reason) if blocked, None otherwise."""
-    if not url:
-        return "empty", "URL is empty"
-
-    parsed_url = urlsplit(url)
-    if parsed_url.scheme.lower() not in url_safety_settings.allowed_schemes:
-        return "scheme", "URL scheme must be http or https"
-
-    hostname = parsed_url.hostname
-    if not hostname:
-        return "host", "URL host is missing or malformed"
-
-    normalized_host = hostname.rstrip(".").lower()
-
-    if normalized_host in url_safety_settings.metadata_hosts:
-        return "metadata", "Target points to a metadata endpoint"
-
-    if (
-        normalized_host in url_safety_settings.localhost_hosts
-        or normalized_host.endswith(".localhost")
-    ):
-        return "localhost", "Target points to a localhost host"
-
-    if normalized_host.endswith(url_safety_settings.service_suffix):
-        return "cluster", "Target points to an internal service host"
-
-    if normalized_host.endswith(url_safety_settings.cluster_local_suffix):
-        return "cluster", "Target points to an internal cluster-local host"
-
-    try:
-        target_ip = ip_address(normalized_host)
-    except ValueError:
-        pass
-    else:
-        if not target_ip.is_global:
-            return "private", "Target points to a private or special-use IP address"
-
-        return None
-
-    if "." not in normalized_host:
-        return "cluster", "Target points to a single-label internal host"
-
-    return None

unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/redirect.py

@@ -1,81 +0,0 @@
-from __future__ import annotations
-
-import logging
-from collections.abc import Awaitable, Callable
-from urllib.parse import urljoin
-
-import httpx
-
-from unique_web_search.services.crawlers.url_safety.models import (
-    BlockedCrawlTarget,
-    CrawlTargetValidationError,
-)
-from unique_web_search.services.crawlers.url_safety.netloc import extract_hostname
-from unique_web_search.services.crawlers.url_safety.settings import url_safety_settings
-
-_LOGGER = logging.getLogger(__name__)
-
-_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})
-
-ValidateUrlFn = Callable[[str], Awaitable[tuple[str, str] | None]]
-
-
-async def resolve_redirect_chain(
-    url: str,
-    validate_url: ValidateUrlFn,
-) -> str:
-    """Follow HTTP 3xx redirects hop-by-hop, validating each destination.
-
-    Returns the final validated URL.
-    Raises CrawlTargetValidationError if any hop is blocked.
-    """
-    current = url
-    timeout = url_safety_settings.redirect_timeout_seconds
-    async with httpx.AsyncClient(follow_redirects=False, timeout=timeout) as client:
-        for _ in range(url_safety_settings.max_redirect_hops):
-            error = await validate_url(current)
-            if error is not None:
-                category, reason = error
-                raise CrawlTargetValidationError(
-                    [
-                        BlockedCrawlTarget(
-                            hostname=extract_hostname(current),
-                            category=category,
-                            reason=reason,
-                        )
-                    ]
-                )
-
-            try:
-                resp = await client.head(current)
-            except Exception as exc:
-                _LOGGER.debug(
-                    "Redirect resolution stopped at %s due to network error: %s",
-                    current,
-                    exc,
-                )
-                break
-
-            if resp.status_code not in _REDIRECT_STATUS_CODES:
-                break
-
-            location = resp.headers.get("location")
-            if not location:
-                break
-
-            current = urljoin(current, location)
-
-    error = await validate_url(current)
-    if error is not None:
-        category, reason = error
-        raise CrawlTargetValidationError(
-            [
-                BlockedCrawlTarget(
-                    hostname=extract_hostname(current),
-                    category=category,
-                    reason=reason,
-                )
-            ]
-        )
-
-    return current

unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/resolver.py

@@ -1,103 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-from ipaddress import ip_address
-from urllib.parse import urlsplit
-
-from unique_web_search.services.crawlers.url_safety import dns
-from unique_web_search.services.crawlers.url_safety.models import (
-    BlockedCrawlTarget,
-    CrawlTargetValidationError,
-    ResolvedCrawlTarget,
-)
-from unique_web_search.services.crawlers.url_safety.netloc import extract_hostname
-from unique_web_search.services.crawlers.url_safety.policy import validate_target_cheap
-
-
-async def resolve_crawl_target(url: str) -> ResolvedCrawlTarget:
-    """Validate and resolve a single URL, performing DNS exactly once."""
-    normalized_url = url.strip()
-
-    validation_error = validate_target_cheap(normalized_url)
-    if validation_error is not None:
-        category, reason = validation_error
-        raise CrawlTargetValidationError(
-            [
-                BlockedCrawlTarget(
-                    hostname=extract_hostname(normalized_url),
-                    category=category,
-                    reason=reason,
-                )
-            ]
-        )
-
-    parsed_url = urlsplit(normalized_url)
-    hostname = parsed_url.hostname
-    if hostname is None:
-        raise CrawlTargetValidationError(
-            [
-                BlockedCrawlTarget(
-                    hostname=None,
-                    category="host",
-                    reason="URL host is missing or malformed",
-                )
-            ]
-        )
-
-    normalized_host = hostname.rstrip(".").lower()
-
-    try:
-        target_ip = ip_address(normalized_host)
-    except ValueError:
-        resolved_addresses, validation_error = await dns.resolve_and_validate_host(
-            normalized_host
-        )
-        if validation_error is not None:
-            category, reason = validation_error
-            raise CrawlTargetValidationError(
-                [
-                    BlockedCrawlTarget(
-                        hostname=normalized_host,
-                        category=category,
-                        reason=reason,
-                    )
-                ]
-            )
-        return ResolvedCrawlTarget(
-            normalized_url=normalized_url,
-            hostname=normalized_host,
-            resolved_ip=resolved_addresses[0],
-            used_dns_resolution=True,
-        )
-
-    return ResolvedCrawlTarget(
-        normalized_url=normalized_url,
-        hostname=normalized_host,
-        resolved_ip=str(target_ip),
-        used_dns_resolution=False,
-    )
-
-
-async def resolve_crawl_targets_batch(urls: list[str]) -> list[ResolvedCrawlTarget]:
-    """Resolve and validate each URL, collecting all policy violations before raising."""
-    results: list[ResolvedCrawlTarget | BaseException] = list(
-        await asyncio.gather(
-            *[resolve_crawl_target(url) for url in urls],
-            return_exceptions=True,
-        )
-    )
-
-    blocked: list[BlockedCrawlTarget] = []
-    resolved: list[ResolvedCrawlTarget] = []
-    for result in results:
-        if isinstance(result, CrawlTargetValidationError):
-            blocked.extend(result.blocked_targets)
-        elif isinstance(result, BaseException):
-            raise result
-        else:
-            resolved.append(result)
-
-    if blocked:
-        raise CrawlTargetValidationError(blocked)
-
-    return resolved