unique-web-search 2026.26.0.dev0__tar.gz → 2026.26.0.dev2__tar.gz

{unique_web_search-2026.26.0.dev0 → unique_web_search-2026.26.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-web-search
-Version: 2026.26.0.dev0
+Version: 2026.26.0.dev2
 Summary: 
 Author: Andreas Hauri, Gustav Hartz, Rami Azouz
 Author-email: Andreas Hauri <andreas@unique.ch>, Gustav Hartz <gustav.hartz.ext@unique.ch>, Rami Azouz <rami.ext@unique.ch>
@@ -17,7 +17,7 @@ Requires-Dist: tavily-python>=0.7.11,<1
 Requires-Dist: unidecode>=1.4.0,<2
 Requires-Dist: azure-ai-projects>=1.0.0,<2
 Requires-Dist: azure-identity>=1.25.0,<2
-Requires-Dist: unique-toolkit[monitoring]>=2026.26.0.dev0,<2026.26.0rc0
+Requires-Dist: unique-toolkit[monitoring]>=2026.26.0.dev2,<2026.26.0rc0
 Requires-Dist: azure-core>=1.36.0,<2
 Requires-Dist: google-cloud-aiplatform>=1.128.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
@@ -28,6 +28,8 @@ Requires-Dist: certifi>=2025.11.12
 Requires-Dist: jinja2>=3.1.6
 Requires-Dist: playwright>=1.58.0
 Requires-Dist: perplexityai>=0.34.0
+Requires-Dist: unique-search-proxy-core>=2026.26.0.dev1,<2026.26.0rc0
+Requires-Dist: unique-search-proxy-sdk>=2026.26.0.dev1,<2026.26.0rc0
 Requires-Python: >=3.12, <4.0
 Description-Content-Type: text/markdown
 

{unique_web_search-2026.26.0.dev0 → unique_web_search-2026.26.0.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_web_search"
-version = "2026.26.0.dev0"
+version = "2026.26.0.dev2"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12,<4.0"
@@ -23,7 +23,7 @@ dependencies = [
     "unidecode>=1.4.0,<2",
     "azure-ai-projects>=1.0.0,<2",
     "azure-identity>=1.25.0,<2",
-    "unique-toolkit[monitoring]>=2026.26.0.dev0,<2026.26.0rc0",
+    "unique-toolkit[monitoring]>=2026.26.0.dev2,<2026.26.0rc0",
     "azure-core>=1.36.0,<2",
     "google-cloud-aiplatform>=1.128.0,<2",
     "google-auth>=2.43.0,<3",
@@ -34,6 +34,8 @@ dependencies = [
     "jinja2>=3.1.6",
     "playwright>=1.58.0",
     "perplexityai>=0.34.0",
+    "unique-search-proxy-core>=2026.26.0.dev1,<2026.26.0rc0",
+    "unique-search-proxy-sdk>=2026.26.0.dev1,<2026.26.0rc0",
 ]
 
 [build-system]
@@ -91,3 +93,5 @@ extend-select = ["I"]
 
 [tool.uv.sources]
 unique-toolkit = { workspace = true }
+unique-search-proxy-sdk = { workspace = true }
+unique-search-proxy-core = { workspace = true }

{unique_web_search-2026.26.0.dev0 → unique_web_search-2026.26.0.dev2}/src/unique_web_search/client_settings.py

@@ -235,3 +235,30 @@ def get_perplexity_search_settings() -> PerplexitySearchSettings:
     if _perplexity_search_settings is None:
         _perplexity_search_settings = PerplexitySearchSettings.from_env_settings()
     return _perplexity_search_settings
+
+
+class SearchProxySettings(BaseModel):
+    base_url: str | None = None
+
+    @property
+    def is_configured(self) -> bool:
+        return self.base_url is not None
+
+    @classmethod
+    def from_env_settings(cls):
+        if env_settings.search_proxy_base_url is None:
+            _LOGGER.warning("Unique Search Proxy missing required setting: Base URL")
+        else:
+            _LOGGER.info("Unique Search Proxy is properly configured")
+
+        return cls(base_url=env_settings.search_proxy_base_url)
+
+
+_search_proxy_settings: SearchProxySettings | None = None
+
+
+def get_search_proxy_settings() -> SearchProxySettings:
+    global _search_proxy_settings
+    if _search_proxy_settings is None:
+        _search_proxy_settings = SearchProxySettings.from_env_settings()
+    return _search_proxy_settings

{unique_web_search-2026.26.0.dev0 → unique_web_search-2026.26.0.dev2}/src/unique_web_search/services/crawlers/__init__.py

@@ -9,6 +9,10 @@ from unique_web_search.services.crawlers.basic import (
     BasicCrawler,
     BasicCrawlerConfig,
 )
+from unique_web_search.services.crawlers.basic_proxy import (
+    BasicCrawlerProxyConfig,
+    BasicProxyCrawler,
+)
 from unique_web_search.services.crawlers.crawl4ai import (
     Crawl4AiCrawler,
     Crawl4AiCrawlerConfig,
@@ -27,11 +31,17 @@ from unique_web_search.services.crawlers.tavily import (
 )
 
 CrawlerTypes = (
-    BasicCrawler | Crawl4AiCrawler | FirecrawlCrawler | JinaCrawler | TavilyCrawler
+    BasicCrawler
+    | BasicProxyCrawler
+    | Crawl4AiCrawler
+    | FirecrawlCrawler
+    | JinaCrawler
+    | TavilyCrawler
 )
 
 CrawlerConfigTypes = (
     BasicCrawlerConfig
+    | BasicCrawlerProxyConfig
     | Crawl4AiCrawlerConfig
     | FirecrawlCrawlerConfig
     | JinaCrawlerConfig
@@ -44,6 +54,7 @@ CRAWLER_NAME_TO_CONFIG = {
     "firecrawl": FirecrawlCrawlerConfig,
     "jina": JinaCrawlerConfig,
     "tavily": TavilyCrawlerConfig,
+    "basic_proxy": BasicCrawlerProxyConfig,
 }
 
 
@@ -61,6 +72,8 @@ def get_crawler_service(
             return FirecrawlCrawler(crawler_config)
         case CrawlerType.JINA:
             return JinaCrawler(crawler_config)
+        case CrawlerType.BASIC_PROXY:
+            return BasicProxyCrawler(crawler_config)
 
 
 def get_crawler_config_types_from_names(crawler_names: list[str]) -> TypeAlias:
@@ -90,6 +103,8 @@ def get_default_crawler_config(
 __all__ = [
     "BasicCrawler",
     "BasicCrawlerConfig",
+    "BasicProxyCrawler",
+    "BasicCrawlerProxyConfig",
     "Crawl4AiCrawler",
     "Crawl4AiCrawlerConfig",
     "FirecrawlCrawler",

{unique_web_search-2026.26.0.dev0 → unique_web_search-2026.26.0.dev2}/src/unique_web_search/services/crawlers/base.py

@@ -20,11 +20,11 @@ from unique_web_search.services.helpers import (
 class CrawlerType(StrEnum):
     CRAWL4AI = "Crawl4AiCrawler"
     BASIC = "BasicCrawler"
+    BASIC_PROXY = "BasicProxyCrawler"
     NO_CRAWLER = "NoCrawler"
     TAVILY = "TavilyCrawler"
     FIRECRAWL = "FirecrawlCrawler"
     JINA = "JinaCrawler"
-    NONE = "None"
 
 
 T = TypeVar("T", bound=CrawlerType)

unique_web_search-2026.26.0.dev2/src/unique_web_search/services/crawlers/basic_proxy.py

@@ -0,0 +1,79 @@
+import logging
+from typing import Literal
+
+from unique_search_proxy_core.crawlers import (
+    BasicCrawlerConfig as BasicCrawlerProxyCoreConfig,
+)
+from unique_search_proxy_core.crawlers.base import CrawlerType as ProxyCrawlerType
+from unique_search_proxy_sdk import UniqueSearchProxyClient
+from unique_search_proxy_sdk._generated.models.crawl_url_result import CrawlUrlResult
+from unique_search_proxy_sdk._generated.models.per_url_error import PerUrlError
+
+from unique_web_search.client_settings import get_search_proxy_settings
+from unique_web_search.services.crawlers.base import BaseCrawlerConfig, CrawlerType
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class BasicCrawlerProxyConfig(
+    BaseCrawlerConfig[CrawlerType.BASIC_PROXY], BasicCrawlerProxyCoreConfig
+):
+    crawler_type: Literal[CrawlerType.BASIC_PROXY] = CrawlerType.BASIC_PROXY
+
+
+class BasicProxyCrawler:
+    """Crawler that delegates fetching to the remote Unique Search Proxy service.
+
+    Intermediate transition crawler: instead of fetching URLs in-process (see
+    :class:`BasicCrawler`), it forwards the request to the search-proxy service
+    via :class:`UniqueSearchProxyClient` and maps the per-URL results back to the
+    markdown list contract shared by the other crawlers.
+    """
+
+    def __init__(self, config: BasicCrawlerProxyConfig):
+        self.config: BasicCrawlerProxyConfig = config
+
+    async def crawl(self, urls: list[str]) -> list[str]:
+        return await self._crawl(urls)
+
+    async def _crawl(self, urls: list[str]) -> list[str]:
+        base_url = get_search_proxy_settings().base_url
+        assert base_url is not None, "Unique Search Proxy base URL is not configured"
+
+        try:
+            async with UniqueSearchProxyClient(
+                base_url=base_url,
+                timeout=float(self.config.timeout),
+            ) as client:
+                response = await client.crawl.crawl(
+                    urls,
+                    crawler_type=ProxyCrawlerType.BASIC,
+                    content_types=self.config.content_types,
+                    timeout=self.config.timeout,
+                    max_concurrent_requests=self.config.max_concurrent_requests,
+                )
+        except Exception as error:
+            _LOGGER.exception("Unique Search Proxy crawl request failed")
+            return [f"Unable to crawl URL via search proxy: {error}" for _ in urls]
+
+        url_to_markdown = {
+            result.url: _result_to_markdown(result) for result in response.results
+        }
+
+        return [
+            url_to_markdown.get(url, "Error: URL not found in search proxy response")
+            for url in urls
+        ]
+
+
+def _result_to_markdown(result: CrawlUrlResult) -> str:
+    if isinstance(result.content, str):
+        return result.content
+
+    if isinstance(result.error, PerUrlError):
+        return f"Error: {result.error.message}"
+
+    if isinstance(result.raw, str):
+        return result.raw
+
+    return "Error: No content returned by search proxy"

{unique_web_search-2026.26.0.dev0 → unique_web_search-2026.26.0.dev2}/src/unique_web_search/settings.py

@@ -38,7 +38,10 @@ class Base(BaseSettings):
     active_search_engines: list[str] = ["google"]
 
     # Default Crawlers
-    active_inhouse_crawlers: list[str] = ["basic", "crawl4ai"]
+    active_inhouse_crawlers: list[str] = ["basic", "crawl4ai", "basic_proxy"]
+
+    # Unique Search Proxy service (used by the basic_proxy crawler)
+    search_proxy_base_url: str | None = None
 
     # Google Search settings
     google_search_api_key: str | None = None