unique_web_search 2026.30.0.dev1__tar.gz → 2026.30.0.dev3__tar.gz

{unique_web_search-2026.30.0.dev1 → unique_web_search-2026.30.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-web-search
-Version: 2026.30.0.dev1
+Version: 2026.30.0.dev3
 Summary: 
 Author: Andreas Hauri, Gustav Hartz, Rami Azouz
 Author-email: Andreas Hauri <andreas@unique.ch>, Gustav Hartz <gustav.hartz.ext@unique.ch>, Rami Azouz <rami.ext@unique.ch>
@@ -11,13 +11,12 @@ Requires-Dist: pydantic-settings>=2.10.1,<3
 Requires-Dist: timeout-decorator>=0.5.0,<1
 Requires-Dist: markdownify>=0.14.1,<1
 Requires-Dist: fake-useragent>=2.2.0,<3
-Requires-Dist: crawl4ai>=0.6.3,<1
 Requires-Dist: firecrawl>=3.3.2,<4
 Requires-Dist: tavily-python>=0.7.11,<1
 Requires-Dist: unidecode>=1.4.0,<2
 Requires-Dist: azure-ai-projects>=1.0.0,<2
 Requires-Dist: azure-identity>=1.25.0,<2
-Requires-Dist: unique-toolkit[monitoring]>=2026.30.0.dev10,<2026.30.0rc0
+Requires-Dist: unique-toolkit[monitoring]>=2026.30.0.dev14,<2026.30.0rc0
 Requires-Dist: azure-core>=1.36.0,<2
 Requires-Dist: google-cloud-aiplatform>=1.128.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
@@ -26,8 +25,7 @@ Requires-Dist: rapidfuzz>=3.0.0,<4
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: certifi>=2025.11.12
 Requires-Dist: jinja2>=3.1.6
-Requires-Dist: playwright>=1.58.0
-Requires-Dist: unique-search-proxy-core>=2026.30.0.dev2,<2026.30.0rc0
+Requires-Dist: unique-search-proxy-core>=2026.30.0.dev3,<2026.30.0rc0
 Requires-Dist: unique-search-proxy-sdk>=2026.30.0.dev2,<2026.30.0rc0
 Requires-Python: >=3.12, <4.0
 Description-Content-Type: text/markdown
@@ -60,7 +58,6 @@ The following diagram illustrates the complete architecture and workflow of the
 
 - **Multiple Web Crawlers**:
   - Basic HTTP Crawler
-  - Crawl4AI
   - Jina Reader
   - Tavily Crawler
   - Firecrawl Crawler
@@ -85,7 +82,7 @@ The following diagram illustrates the complete architecture and workflow of the
 For deeper dives into each subsystem, see the dedicated READMEs:
 
 - [Search Engines](./unique_web_search/services/search_engine/README.md) &mdash; full catalogue of supported engines, configuration, and usage examples.
-- [Crawlers](./unique_web_search/services/crawlers/README.md) &mdash; comparison of crawling strategies (Basic, Crawl4AI, Tavily, Firecrawl, Jina) with setup guides.
+- [Crawlers](./unique_web_search/services/crawlers/README.md) &mdash; comparison of crawling strategies (Basic, Tavily, Firecrawl, Jina) with setup guides.
 - [Executors](./unique_web_search/services/executors/README.md) &mdash; orchestration layer (V1 & V2) covering query refinement, planning, logging, and best practices.
 
 ## Configuration

{unique_web_search-2026.30.0.dev1 → unique_web_search-2026.30.0.dev3}/README.md

@@ -26,7 +26,6 @@ The following diagram illustrates the complete architecture and workflow of the
 
 - **Multiple Web Crawlers**:
   - Basic HTTP Crawler
-  - Crawl4AI
   - Jina Reader
   - Tavily Crawler
   - Firecrawl Crawler
@@ -51,7 +50,7 @@ The following diagram illustrates the complete architecture and workflow of the
 For deeper dives into each subsystem, see the dedicated READMEs:
 
 - [Search Engines](./unique_web_search/services/search_engine/README.md) — full catalogue of supported engines, configuration, and usage examples.
-- [Crawlers](./unique_web_search/services/crawlers/README.md) — comparison of crawling strategies (Basic, Crawl4AI, Tavily, Firecrawl, Jina) with setup guides.
+- [Crawlers](./unique_web_search/services/crawlers/README.md) — comparison of crawling strategies (Basic, Tavily, Firecrawl, Jina) with setup guides.
 - [Executors](./unique_web_search/services/executors/README.md) — orchestration layer (V1 & V2) covering query refinement, planning, logging, and best practices.
 
 ## Configuration

{unique_web_search-2026.30.0.dev1 → unique_web_search-2026.30.0.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_web_search"
-version = "2026.30.0.dev1"
+version = "2026.30.0.dev3"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12,<4.0"
@@ -17,13 +17,12 @@ dependencies = [
     "timeout-decorator>=0.5.0,<1",
     "markdownify>=0.14.1,<1",
     "fake-useragent>=2.2.0,<3",
-    "crawl4ai>=0.6.3,<1",
     "firecrawl>=3.3.2,<4",
     "tavily-python>=0.7.11,<1",
     "unidecode>=1.4.0,<2",
     "azure-ai-projects>=1.0.0,<2",
     "azure-identity>=1.25.0,<2",
-    "unique-toolkit[monitoring]>=2026.30.0.dev10,<2026.30.0rc0",
+    "unique-toolkit[monitoring]>=2026.30.0.dev14,<2026.30.0rc0",
     "azure-core>=1.36.0,<2",
     "google-cloud-aiplatform>=1.128.0,<2",
     "google-auth>=2.43.0,<3",
@@ -32,8 +31,7 @@ dependencies = [
     "httpx>=0.28.1",
     "certifi>=2025.11.12",
     "jinja2>=3.1.6",
-    "playwright>=1.58.0",
-    "unique-search-proxy-core>=2026.30.0.dev2,<2026.30.0rc0",
+    "unique-search-proxy-core>=2026.30.0.dev3,<2026.30.0rc0",
     "unique-search-proxy-sdk>=2026.30.0.dev2,<2026.30.0rc0",
 ]
 

{unique_web_search-2026.30.0.dev1 → unique_web_search-2026.30.0.dev3}/src/unique_web_search/config.py

@@ -180,7 +180,7 @@ class WebSearchConfig(BaseToolConfig):
         default_factory=DefaultCrawler,  # type: ignore (This type is computed at runtime so pyright is not able to infer it)
         title="Web Page Reader",
         description="Choose and configure how web pages are fetched and converted to readable text.",
-        discriminator="crawler_type",
+        discriminator="crawler",
     )
 
     content_processor_config: ContentProcessorConfig = Field(

{unique_web_search-2026.30.0.dev1 → unique_web_search-2026.30.0.dev3}/src/unique_web_search/service.py

@@ -1,9 +1,7 @@
 import logging
-import re
 from datetime import datetime
 from time import time
 
-from jinja2 import Template
 from typing_extensions import override
 from unique_search_proxy_core.param_policy.exposed_params import ExposedParams
 from unique_search_proxy_core.search_engines.base import BaseSearchEngineConfig
@@ -33,29 +31,22 @@ from unique_web_search.services.argument_screening import (
 )
 from unique_web_search.services.content_processing import ContentProcessor
 from unique_web_search.services.crawlers import get_crawler_service
-from unique_web_search.services.executors import (
-    WebSearchMode,
-    WebSearchV1Executor,
-    WebSearchV2Executor,
-    WebSearchV3Config,
-    WebSearchV3Executor,
-)
 from unique_web_search.services.executors.context import (
     ExecutorCallbacks,
     ExecutorConfiguration,
     ExecutorServiceContext,
 )
-from unique_web_search.services.executors.v1.schema import WebSearchToolParameters
-from unique_web_search.services.executors.v2.schema import WebSearchPlan
-from unique_web_search.services.executors.v3.schema import WebSearchV3ToolParameters
+from unique_web_search.services.executors.modes import (
+    WebSearchExecutor,
+    WebSearchToolContext,
+    WebSearchToolParametersInstance,
+    get_mode_strategy,
+)
 from unique_web_search.services.message_log import WebSearchMessageLogger
 from unique_web_search.services.query_elicitation import QueryElicitationService
 from unique_web_search.services.search_engine import (
-    SearchEngineMode,
-    get_search_engine_mode,
     get_search_engine_service,
 )
-from unique_web_search.services.search_engine.custom_api import CustomAPIConfig
 from unique_web_search.utils import (
     WebPageChunk,
     reduce_sources_to_token_limit,
@@ -100,6 +91,7 @@ class WebSearchTool(Tool[WebSearchConfig]):
         self.debug = self.config.debug
         self._display_name = kwargs.get("display_name", "Web Search")
         self.exposed_params_cls = self._resolve_exposed_params_cls()
+        self._mode_strategy = get_mode_strategy(self.config.web_search_mode_config)
 
         def content_reducer(web_page_chunks: list[WebPageChunk]) -> list[WebPageChunk]:
 
@@ -121,85 +113,30 @@ class WebSearchTool(Tool[WebSearchConfig]):
             return cfg.exposed_params_model()
         return None
 
-    def _resolve_search_engine_mode(self) -> SearchEngineMode:
-        """Derive the search-engine mode, respecting CustomAPI overrides."""
-        cfg = self.search_engine_service.config
-        override = cfg.search_engine_mode if isinstance(cfg, CustomAPIConfig) else None
-        return get_search_engine_mode(cfg.engine, override=override)
+    def _tool_context(self) -> WebSearchToolContext:
+        return WebSearchToolContext(
+            search_engine_config=self.search_engine_service.config,
+            date_string=datetime.now().strftime("%A %B %d, %Y"),
+            exposed_params_cls=self.exposed_params_cls,
+        )
 
     @override
     def tool_description(self) -> LanguageModelToolDescription:
-        exposed = self.exposed_params_cls
-        if self.config.web_search_mode_config.mode == WebSearchMode.V1:
-            self.tool_parameter_calls = WebSearchToolParameters.with_exposed_params(
-                exposed
-            )
-        else:
-            if self.config.web_search_mode_config.mode == WebSearchMode.V3:
-                self.tool_parameter_calls = (
-                    WebSearchV3ToolParameters.with_exposed_params(exposed)
-                )
-            else:
-                engine_mode = self._resolve_search_engine_mode()
-                self.tool_parameter_calls = WebSearchPlan.with_search_engine_mode(
-                    engine_mode
-                )
-
-        tool_description = self.config.web_search_mode_config.tool_description
-
-        if self.config.web_search_mode_config.mode == WebSearchMode.V3:
-            tool_description = Template(tool_description).render(
-                tool_parameters_schema=WebSearchV3ToolParameters.schema_hint(exposed),
-            )
-
+        ctx = self._tool_context()
+        self.tool_parameter_calls = self._mode_strategy.build_tool_parameters(ctx)
         return LanguageModelToolDescription(
             name=self.name,
-            description=tool_description,
+            description=self._mode_strategy.tool_description(ctx),
             parameters=self.tool_parameter_calls,
         )
 
     def tool_description_for_system_prompt(self) -> str:
-        mode_config = self.config.web_search_mode_config
-        if mode_config.mode == WebSearchMode.V1:
-            return mode_config.tool_description_for_system_prompt
-
-        template_str = mode_config.tool_description_for_system_prompt
-
-        if mode_config.mode == WebSearchMode.V3:
-            return Template(template_str).render(
-                date_string=datetime.now().strftime("%A %B %d, %Y"),
-            )
-
-        # Backwards compatibility: V2 prompts persisted before the Jinja
-        # migration use the legacy ``$max_steps`` placeholder. Jinja would
-        # otherwise pass it through verbatim, so rewrite it to the Jinja
-        # equivalent before rendering.
-        if "$max_steps" in template_str:
-            _LOGGER.warning(
-                "V2 web-search prompt contains legacy '$max_steps' placeholder; "
-                "rewriting to '{{ max_steps }}'. Please update the stored "
-                "configuration to use Jinja syntax."
-            )
-            template_str = template_str.replace("$max_steps", "{{ max_steps }}")
-
-        engine_mode = self._resolve_search_engine_mode()
-        return Template(template_str).render(
-            max_steps=mode_config.max_steps,
-            date_string=datetime.now().strftime("%A %B %d, %Y"),
-            search_engine_mode=engine_mode.value,
-            tool_parameters_schema=WebSearchPlan.schema_hint(engine_mode),
-            example_simple=WebSearchPlan.build_example_simple(
-                engine_mode
-            ).model_dump_json(indent=2),
-            example_complex=WebSearchPlan.build_example_complex(
-                engine_mode
-            ).model_dump_json(indent=2),
-        )
+        return self._mode_strategy.system_prompt(self._tool_context())
 
     def tool_format_information_for_system_prompt(self) -> str:
-        if self.config.web_search_active_mode == WebSearchMode.V3:
-            return self.config.web_search_mode_config_v3.tool_format_information_for_system_prompt
-        return self.config.tool_format_information_for_system_prompt
+        return self._mode_strategy.format_information_for_system_prompt(
+            default=self.config.tool_format_information_for_system_prompt,
+        )
 
     def evaluation_check_list(self) -> list[EvaluationMetricName]:
         return self.config.evaluation_check_list
@@ -301,18 +238,16 @@ class WebSearchTool(Tool[WebSearchConfig]):
     def _get_executor(
         self,
         tool_call: LanguageModelFunction,
-        parameters: WebSearchPlan | WebSearchToolParameters | WebSearchV3ToolParameters,
+        parameters: WebSearchToolParametersInstance,
         debug_info: WebSearchDebugInfo,
         web_search_message_logger: WebSearchMessageLogger,
-    ) -> WebSearchV1Executor | WebSearchV2Executor | WebSearchV3Executor:
-        # Initialize query elicitation service and get callbacks
+    ) -> WebSearchExecutor:
         elicitation_service = QueryElicitationService(
             chat_service=self._chat_service,
             display_name=self.display_name(),
             config=self.config.query_elicitation_config,
         )
 
-        # Build context objects
         services = ExecutorServiceContext(
             search_engine_service=self.search_engine_service,
             crawler_service=self.crawler_service,
@@ -334,46 +269,14 @@ class WebSearchTool(Tool[WebSearchConfig]):
             tool_progress_reporter=self._ff_tool_progress_reporter(),
         )
 
-        search_config = self.config.web_search_mode_config
-        if search_config.mode == WebSearchMode.V3:
-            assert isinstance(search_config, WebSearchV3Config)
-            assert isinstance(parameters, WebSearchV3ToolParameters)
-
-            return WebSearchV3Executor(
-                services=services,
-                config=config,
-                callbacks=callbacks,
-                tool_call=tool_call,
-                tool_parameters=parameters,
-                exposed_params_cls=self.exposed_params_cls,
-            )
-        if isinstance(parameters, WebSearchPlan):
-            assert search_config.mode == WebSearchMode.V2
-            return WebSearchV2Executor(
-                services=services,
-                config=config,
-                callbacks=callbacks,
-                tool_call=tool_call,
-                tool_parameters=parameters,
-                max_steps=search_config.max_steps,
-                exposed_params_cls=self.exposed_params_cls,
-            )
-        elif isinstance(parameters, WebSearchToolParameters):
-            assert search_config.mode == WebSearchMode.V1
-            return WebSearchV1Executor(
-                services=services,
-                config=config,
-                callbacks=callbacks,
-                tool_call=tool_call,
-                tool_parameters=parameters,
-                refine_query_system_prompt=search_config.refine_query_mode.system_prompt,
-                refine_query_language_model=search_config.refine_query_mode.language_model,
-                mode=search_config.refine_query_mode.mode,
-                max_queries=search_config.max_queries,
-                exposed_params_cls=self.exposed_params_cls,
-            )
-        else:
-            raise ValueError(f"Invalid parameters: {parameters}")
+        return self._mode_strategy.build_executor(
+            services=services,
+            config=config,
+            callbacks=callbacks,
+            tool_call=tool_call,
+            parameters=parameters,
+            exposed_params_cls=self.exposed_params_cls,
+        )
 
     def get_evaluation_checks_based_on_tool_response(
         self,
@@ -429,20 +332,12 @@ class WebSearchTool(Tool[WebSearchConfig]):
 
     def _build_display_name(
         self,
-        parameters: WebSearchPlan | WebSearchToolParameters | WebSearchV3ToolParameters,
+        parameters: WebSearchToolParametersInstance,
     ) -> str:
-        if not isinstance(parameters, WebSearchV3ToolParameters):
-            return self.display_name()
-
-        display_name = self.display_name()
-        # Remove "search" from the display name (case-insensitive, all occurrences).
-        display_name = re.sub(
-            r"\bsearch\b", "", display_name, flags=re.IGNORECASE
-        ).strip()
-
-        suffix = parameters.get_display_name_suffix()
-
-        return display_name + suffix
+        return self._mode_strategy.build_display_name(
+            base_display_name=self.display_name(),
+            parameters=parameters,
+        )
 
 
 ToolFactory.register_tool(WebSearchTool, WebSearchConfig)

unique_web_search-2026.30.0.dev3/src/unique_web_search/services/crawlers/README.md

@@ -0,0 +1,58 @@
+# Crawlers
+
+Web-page readers used by Unique Web Search to fetch and convert URLs into readable text.
+
+## Architecture
+
+Crawler configuration lives in **proxy-core** (`unique_search_proxy_core.crawlers`). The tool consumes those configs, registers implementations via `CRAWLER_REGISTRY`, and centralizes proxy dispatch in `BaseCrawler` — the same pattern as search engines.
+
+```mermaid
+flowchart LR
+  Config["proxy-core *Config\n(crawler discriminator)"]
+  Registry["CRAWLER_REGISTRY"]
+  Base["BaseCrawler._proxy_crawl"]
+  SDK["client.crawl.crawl(crawler, urls, **config)"]
+  Legacy["_legacy_crawl\n(direct provider call)"]
+
+  Config --> Registry
+  Registry --> Base
+  Base -->|"proxy enabled"| SDK
+  Base -->|"proxy disabled"| Legacy
+```
+
+### Key invariants
+
+- **No exposable params.** Crawlers receive URLs only; deployment defaults come from the config. There is no LLM override surface.
+- **Proxy request = config fields + urls.** `BaseCrawler._proxy_crawl` dumps `config.model_dump(exclude={"crawler"})` into the generic SDK call. No merge helper.
+- **Legacy fallback retained.** When the search proxy is disabled, each crawler implements `_legacy_crawl` against the provider SDK / HTTP client.
+
+## Available crawlers
+
+| Name | Config | Auth | Notes |
+|------|--------|------|-------|
+| Basic | tool-local `BasicConfig` (extends proxy-core) | None | HTTP fetch + markdownify; content-type toggles; `url_blocked_patterns` (to be decommissioned) |
+| Tavily | `TavilyConfig` | `TAVILY_API_KEY` | Extract API; `extract_depth`, `format`, … |
+| Jina | `JinaConfig` | `JINA_API_KEY` | Reader API; typed `return_format` / `engine` / `do_not_track` |
+| Firecrawl | `FirecrawlConfig` | `FIRECRAWL_API_KEY` | Batch scrape; `only_main_content`, … |
+
+## Configuration
+
+`WebSearchConfig.crawler_config` is a discriminated union on `crawler` (`"Basic"` / `"Tavily"` / `"Jina"` / `"Firecrawl"`). Active crawlers are selected via `active_inhouse_crawlers` plus API-key auto-append in settings.
+
+### Example
+
+```python
+from unique_search_proxy_core.crawlers import CrawlerType
+from unique_web_search.services.crawlers import BasicConfig, get_crawler_service
+
+config = BasicConfig(crawler=CrawlerType.BASIC, timeout=30)
+crawler = get_crawler_service(config)
+markdowns = await crawler.crawl(["https://example.com"])
+```
+
+## Adding a crawler
+
+1. Define the deployment config + request model in proxy-core (`unique_search_proxy_core.crawlers.<name>`).
+2. Add a tool module under `services/crawlers/` that `@register_crawler(...)`s an implementation with `_legacy_crawl` only.
+3. Modules are auto-discovered — no `__init__.py` factory edit needed.
+4. Add the config/impl types to the static unions in `__init__.py` (for pyright) and cover them in `tests/test_crawler_registry.py`.

unique_web_search-2026.30.0.dev3/src/unique_web_search/services/crawlers/__init__.py

@@ -0,0 +1,60 @@
+from pydantic import BaseModel
+from unique_search_proxy_core.crawlers import CrawlerType
+from unique_search_proxy_core.crawlers.firecrawl.schema import FirecrawlConfig
+from unique_search_proxy_core.crawlers.jina.schema import JinaConfig
+from unique_search_proxy_core.crawlers.tavily.schema import TavilyConfig
+
+from unique_web_search.services.crawlers.base import BaseCrawler
+from unique_web_search.services.crawlers.registry import (
+    CRAWLER_REGISTRY,
+    get_crawler_service,
+)
+
+CRAWLER_REGISTRY.autodiscover(
+    __path__,
+    __name__,
+    exclude=frozenset({"url_safety", "utils", "registry"}),
+)
+
+from unique_web_search.services.crawlers.basic import (  # noqa: E402
+    BasicConfig,
+    BasicCrawler,
+)
+from unique_web_search.services.crawlers.firecrawl import (  # noqa: E402
+    FirecrawlCrawler,
+)
+from unique_web_search.services.crawlers.jina import JinaCrawler  # noqa: E402
+from unique_web_search.services.crawlers.tavily import TavilyCrawler  # noqa: E402
+
+CrawlerTypes = BasicCrawler | FirecrawlCrawler | JinaCrawler | TavilyCrawler
+
+CrawlerConfigTypes = BasicConfig | FirecrawlConfig | JinaConfig | TavilyConfig
+
+CRAWLER_NAME_TO_CONFIG = CRAWLER_REGISTRY.name_to_config()
+
+
+def get_crawler_config_types_from_names(crawler_names: list[str]) -> type[BaseModel]:
+    return CRAWLER_REGISTRY.config_types_from_names(crawler_names)
+
+
+def get_default_crawler_config(crawler_names: list[str]) -> type[BaseModel]:
+    return CRAWLER_REGISTRY.default_config(crawler_names)
+
+
+__all__ = [
+    "BasicCrawler",
+    "BasicConfig",
+    "FirecrawlCrawler",
+    "FirecrawlConfig",
+    "JinaCrawler",
+    "JinaConfig",
+    "TavilyCrawler",
+    "TavilyConfig",
+    "CrawlerType",
+    "BaseCrawler",
+    "get_crawler_service",
+    "CrawlerConfigTypes",
+    "CrawlerTypes",
+    "get_crawler_config_types_from_names",
+    "get_default_crawler_config",
+]

unique_web_search-2026.30.0.dev3/src/unique_web_search/services/crawlers/base.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Generic, TypeVar
+
+from unique_search_proxy_core.crawlers.base import BaseCrawlerConfig
+from unique_search_proxy_core.url_safety import (
+    CrawlTargetValidationError,
+    ResolvedCrawlTarget,
+    UrlSafetyService,
+)
+
+from unique_web_search.metrics import crawl_blocked
+from unique_web_search.services.proxy.bridge import (
+    open_search_proxy_client,
+    search_proxy_client_enabled,
+)
+from unique_web_search.services.proxy.mappers import map_crawl_response
+
+CrawlerConfig = TypeVar(
+    "CrawlerConfig",
+    bound=BaseCrawlerConfig,
+)
+
+
+class BaseCrawler(ABC, Generic[CrawlerConfig]):
+    """Base class for web-page crawlers.
+
+    The search-proxy path is implemented entirely here. Subclasses only provide
+    the direct ``_legacy_crawl`` implementation used when the proxy is disabled.
+    """
+
+    def __init__(self, config: CrawlerConfig):
+        self.config = config
+
+    async def crawl(self, urls: list[str]) -> list[str]:
+        if search_proxy_client_enabled:
+            return await self._proxy_crawl(urls)
+
+        try:
+            targets = await UrlSafetyService.validate_batch_urls(urls)
+        except CrawlTargetValidationError as exc:
+            for target in exc.blocked_targets:
+                crawl_blocked.labels(reason_category=target.category).inc()
+            raise
+        return await self._legacy_crawl(targets)
+
+    async def _proxy_crawl(self, urls: list[str]) -> list[str]:
+        """Dump deployment config fields into the generic proxy crawl call."""
+        params = self.config.model_dump(exclude={"crawler"}, exclude_none=True)
+        async with open_search_proxy_client(
+            timeout=float(self.config.timeout)
+        ) as client:
+            response = await client.crawl.crawl(
+                urls=urls,
+                crawler=self.config.crawler,
+                **params,
+            )
+        return map_crawl_response(response, urls)
+
+    @abstractmethod
+    async def _legacy_crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]: ...