unique_web_search 2026.30.0.dev2__tar.gz → 2026.30.0.dev3__tar.gz

{unique_web_search-2026.30.0.dev2 → unique_web_search-2026.30.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-web-search
-Version: 2026.30.0.dev2
+Version: 2026.30.0.dev3
 Summary: 
 Author: Andreas Hauri, Gustav Hartz, Rami Azouz
 Author-email: Andreas Hauri <andreas@unique.ch>, Gustav Hartz <gustav.hartz.ext@unique.ch>, Rami Azouz <rami.ext@unique.ch>
@@ -11,13 +11,12 @@ Requires-Dist: pydantic-settings>=2.10.1,<3
 Requires-Dist: timeout-decorator>=0.5.0,<1
 Requires-Dist: markdownify>=0.14.1,<1
 Requires-Dist: fake-useragent>=2.2.0,<3
-Requires-Dist: crawl4ai>=0.6.3,<1
 Requires-Dist: firecrawl>=3.3.2,<4
 Requires-Dist: tavily-python>=0.7.11,<1
 Requires-Dist: unidecode>=1.4.0,<2
 Requires-Dist: azure-ai-projects>=1.0.0,<2
 Requires-Dist: azure-identity>=1.25.0,<2
-Requires-Dist: unique-toolkit[monitoring]>=2026.30.0.dev12,<2026.30.0rc0
+Requires-Dist: unique-toolkit[monitoring]>=2026.30.0.dev14,<2026.30.0rc0
 Requires-Dist: azure-core>=1.36.0,<2
 Requires-Dist: google-cloud-aiplatform>=1.128.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
@@ -26,8 +25,7 @@ Requires-Dist: rapidfuzz>=3.0.0,<4
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: certifi>=2025.11.12
 Requires-Dist: jinja2>=3.1.6
-Requires-Dist: playwright>=1.58.0
-Requires-Dist: unique-search-proxy-core>=2026.30.0.dev2,<2026.30.0rc0
+Requires-Dist: unique-search-proxy-core>=2026.30.0.dev3,<2026.30.0rc0
 Requires-Dist: unique-search-proxy-sdk>=2026.30.0.dev2,<2026.30.0rc0
 Requires-Python: >=3.12, <4.0
 Description-Content-Type: text/markdown
@@ -60,7 +58,6 @@ The following diagram illustrates the complete architecture and workflow of the
 
 - **Multiple Web Crawlers**:
   - Basic HTTP Crawler
-  - Crawl4AI
   - Jina Reader
   - Tavily Crawler
   - Firecrawl Crawler
@@ -85,7 +82,7 @@ The following diagram illustrates the complete architecture and workflow of the
 For deeper dives into each subsystem, see the dedicated READMEs:
 
 - [Search Engines](./unique_web_search/services/search_engine/README.md) &mdash; full catalogue of supported engines, configuration, and usage examples.
-- [Crawlers](./unique_web_search/services/crawlers/README.md) &mdash; comparison of crawling strategies (Basic, Crawl4AI, Tavily, Firecrawl, Jina) with setup guides.
+- [Crawlers](./unique_web_search/services/crawlers/README.md) &mdash; comparison of crawling strategies (Basic, Tavily, Firecrawl, Jina) with setup guides.
 - [Executors](./unique_web_search/services/executors/README.md) &mdash; orchestration layer (V1 & V2) covering query refinement, planning, logging, and best practices.
 
 ## Configuration

{unique_web_search-2026.30.0.dev2 → unique_web_search-2026.30.0.dev3}/README.md

@@ -26,7 +26,6 @@ The following diagram illustrates the complete architecture and workflow of the
 
 - **Multiple Web Crawlers**:
   - Basic HTTP Crawler
-  - Crawl4AI
   - Jina Reader
   - Tavily Crawler
   - Firecrawl Crawler
@@ -51,7 +50,7 @@ The following diagram illustrates the complete architecture and workflow of the
 For deeper dives into each subsystem, see the dedicated READMEs:
 
 - [Search Engines](./unique_web_search/services/search_engine/README.md) — full catalogue of supported engines, configuration, and usage examples.
-- [Crawlers](./unique_web_search/services/crawlers/README.md) — comparison of crawling strategies (Basic, Crawl4AI, Tavily, Firecrawl, Jina) with setup guides.
+- [Crawlers](./unique_web_search/services/crawlers/README.md) — comparison of crawling strategies (Basic, Tavily, Firecrawl, Jina) with setup guides.
 - [Executors](./unique_web_search/services/executors/README.md) — orchestration layer (V1 & V2) covering query refinement, planning, logging, and best practices.
 
 ## Configuration

{unique_web_search-2026.30.0.dev2 → unique_web_search-2026.30.0.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_web_search"
-version = "2026.30.0.dev2"
+version = "2026.30.0.dev3"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12,<4.0"
@@ -17,13 +17,12 @@ dependencies = [
     "timeout-decorator>=0.5.0,<1",
     "markdownify>=0.14.1,<1",
     "fake-useragent>=2.2.0,<3",
-    "crawl4ai>=0.6.3,<1",
     "firecrawl>=3.3.2,<4",
     "tavily-python>=0.7.11,<1",
     "unidecode>=1.4.0,<2",
     "azure-ai-projects>=1.0.0,<2",
     "azure-identity>=1.25.0,<2",
-    "unique-toolkit[monitoring]>=2026.30.0.dev12,<2026.30.0rc0",
+    "unique-toolkit[monitoring]>=2026.30.0.dev14,<2026.30.0rc0",
     "azure-core>=1.36.0,<2",
     "google-cloud-aiplatform>=1.128.0,<2",
     "google-auth>=2.43.0,<3",
@@ -32,8 +31,7 @@ dependencies = [
     "httpx>=0.28.1",
     "certifi>=2025.11.12",
     "jinja2>=3.1.6",
-    "playwright>=1.58.0",
-    "unique-search-proxy-core>=2026.30.0.dev2,<2026.30.0rc0",
+    "unique-search-proxy-core>=2026.30.0.dev3,<2026.30.0rc0",
     "unique-search-proxy-sdk>=2026.30.0.dev2,<2026.30.0rc0",
 ]
 

{unique_web_search-2026.30.0.dev2 → unique_web_search-2026.30.0.dev3}/src/unique_web_search/config.py

@@ -180,7 +180,7 @@ class WebSearchConfig(BaseToolConfig):
         default_factory=DefaultCrawler,  # type: ignore (This type is computed at runtime so pyright is not able to infer it)
         title="Web Page Reader",
         description="Choose and configure how web pages are fetched and converted to readable text.",
-        discriminator="crawler_type",
+        discriminator="crawler",
     )
 
     content_processor_config: ContentProcessorConfig = Field(

unique_web_search-2026.30.0.dev3/src/unique_web_search/services/crawlers/README.md

@@ -0,0 +1,58 @@
+# Crawlers
+
+Web-page readers used by Unique Web Search to fetch and convert URLs into readable text.
+
+## Architecture
+
+Crawler configuration lives in **proxy-core** (`unique_search_proxy_core.crawlers`). The tool consumes those configs, registers implementations via `CRAWLER_REGISTRY`, and centralizes proxy dispatch in `BaseCrawler` — the same pattern as search engines.
+
+```mermaid
+flowchart LR
+  Config["proxy-core *Config\n(crawler discriminator)"]
+  Registry["CRAWLER_REGISTRY"]
+  Base["BaseCrawler._proxy_crawl"]
+  SDK["client.crawl.crawl(crawler, urls, **config)"]
+  Legacy["_legacy_crawl\n(direct provider call)"]
+
+  Config --> Registry
+  Registry --> Base
+  Base -->|"proxy enabled"| SDK
+  Base -->|"proxy disabled"| Legacy
+```
+
+### Key invariants
+
+- **No exposable params.** Crawlers receive URLs only; deployment defaults come from the config. There is no LLM override surface.
+- **Proxy request = config fields + urls.** `BaseCrawler._proxy_crawl` dumps `config.model_dump(exclude={"crawler"})` into the generic SDK call. No merge helper.
+- **Legacy fallback retained.** When the search proxy is disabled, each crawler implements `_legacy_crawl` against the provider SDK / HTTP client.
+
+## Available crawlers
+
+| Name | Config | Auth | Notes |
+|------|--------|------|-------|
+| Basic | tool-local `BasicConfig` (extends proxy-core) | None | HTTP fetch + markdownify; content-type toggles; `url_blocked_patterns` (to be decommissioned) |
+| Tavily | `TavilyConfig` | `TAVILY_API_KEY` | Extract API; `extract_depth`, `format`, … |
+| Jina | `JinaConfig` | `JINA_API_KEY` | Reader API; typed `return_format` / `engine` / `do_not_track` |
+| Firecrawl | `FirecrawlConfig` | `FIRECRAWL_API_KEY` | Batch scrape; `only_main_content`, … |
+
+## Configuration
+
+`WebSearchConfig.crawler_config` is a discriminated union on `crawler` (`"Basic"` / `"Tavily"` / `"Jina"` / `"Firecrawl"`). Active crawlers are selected via `active_inhouse_crawlers` plus API-key auto-append in settings.
+
+### Example
+
+```python
+from unique_search_proxy_core.crawlers import CrawlerType
+from unique_web_search.services.crawlers import BasicConfig, get_crawler_service
+
+config = BasicConfig(crawler=CrawlerType.BASIC, timeout=30)
+crawler = get_crawler_service(config)
+markdowns = await crawler.crawl(["https://example.com"])
+```
+
+## Adding a crawler
+
+1. Define the deployment config + request model in proxy-core (`unique_search_proxy_core.crawlers.<name>`).
+2. Add a tool module under `services/crawlers/` that `@register_crawler(...)`s an implementation with `_legacy_crawl` only.
+3. Modules are auto-discovered — no `__init__.py` factory edit needed.
+4. Add the config/impl types to the static unions in `__init__.py` (for pyright) and cover them in `tests/test_crawler_registry.py`.

unique_web_search-2026.30.0.dev3/src/unique_web_search/services/crawlers/__init__.py

@@ -0,0 +1,60 @@
+from pydantic import BaseModel
+from unique_search_proxy_core.crawlers import CrawlerType
+from unique_search_proxy_core.crawlers.firecrawl.schema import FirecrawlConfig
+from unique_search_proxy_core.crawlers.jina.schema import JinaConfig
+from unique_search_proxy_core.crawlers.tavily.schema import TavilyConfig
+
+from unique_web_search.services.crawlers.base import BaseCrawler
+from unique_web_search.services.crawlers.registry import (
+    CRAWLER_REGISTRY,
+    get_crawler_service,
+)
+
+CRAWLER_REGISTRY.autodiscover(
+    __path__,
+    __name__,
+    exclude=frozenset({"url_safety", "utils", "registry"}),
+)
+
+from unique_web_search.services.crawlers.basic import (  # noqa: E402
+    BasicConfig,
+    BasicCrawler,
+)
+from unique_web_search.services.crawlers.firecrawl import (  # noqa: E402
+    FirecrawlCrawler,
+)
+from unique_web_search.services.crawlers.jina import JinaCrawler  # noqa: E402
+from unique_web_search.services.crawlers.tavily import TavilyCrawler  # noqa: E402
+
+CrawlerTypes = BasicCrawler | FirecrawlCrawler | JinaCrawler | TavilyCrawler
+
+CrawlerConfigTypes = BasicConfig | FirecrawlConfig | JinaConfig | TavilyConfig
+
+CRAWLER_NAME_TO_CONFIG = CRAWLER_REGISTRY.name_to_config()
+
+
+def get_crawler_config_types_from_names(crawler_names: list[str]) -> type[BaseModel]:
+    return CRAWLER_REGISTRY.config_types_from_names(crawler_names)
+
+
+def get_default_crawler_config(crawler_names: list[str]) -> type[BaseModel]:
+    return CRAWLER_REGISTRY.default_config(crawler_names)
+
+
+__all__ = [
+    "BasicCrawler",
+    "BasicConfig",
+    "FirecrawlCrawler",
+    "FirecrawlConfig",
+    "JinaCrawler",
+    "JinaConfig",
+    "TavilyCrawler",
+    "TavilyConfig",
+    "CrawlerType",
+    "BaseCrawler",
+    "get_crawler_service",
+    "CrawlerConfigTypes",
+    "CrawlerTypes",
+    "get_crawler_config_types_from_names",
+    "get_default_crawler_config",
+]

unique_web_search-2026.30.0.dev3/src/unique_web_search/services/crawlers/base.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Generic, TypeVar
+
+from unique_search_proxy_core.crawlers.base import BaseCrawlerConfig
+from unique_search_proxy_core.url_safety import (
+    CrawlTargetValidationError,
+    ResolvedCrawlTarget,
+    UrlSafetyService,
+)
+
+from unique_web_search.metrics import crawl_blocked
+from unique_web_search.services.proxy.bridge import (
+    open_search_proxy_client,
+    search_proxy_client_enabled,
+)
+from unique_web_search.services.proxy.mappers import map_crawl_response
+
+CrawlerConfig = TypeVar(
+    "CrawlerConfig",
+    bound=BaseCrawlerConfig,
+)
+
+
+class BaseCrawler(ABC, Generic[CrawlerConfig]):
+    """Base class for web-page crawlers.
+
+    The search-proxy path is implemented entirely here. Subclasses only provide
+    the direct ``_legacy_crawl`` implementation used when the proxy is disabled.
+    """
+
+    def __init__(self, config: CrawlerConfig):
+        self.config = config
+
+    async def crawl(self, urls: list[str]) -> list[str]:
+        if search_proxy_client_enabled:
+            return await self._proxy_crawl(urls)
+
+        try:
+            targets = await UrlSafetyService.validate_batch_urls(urls)
+        except CrawlTargetValidationError as exc:
+            for target in exc.blocked_targets:
+                crawl_blocked.labels(reason_category=target.category).inc()
+            raise
+        return await self._legacy_crawl(targets)
+
+    async def _proxy_crawl(self, urls: list[str]) -> list[str]:
+        """Dump deployment config fields into the generic proxy crawl call."""
+        params = self.config.model_dump(exclude={"crawler"}, exclude_none=True)
+        async with open_search_proxy_client(
+            timeout=float(self.config.timeout)
+        ) as client:
+            response = await client.crawl.crawl(
+                urls=urls,
+                crawler=self.config.crawler,
+                **params,
+            )
+        return map_crawl_response(response, urls)
+
+    @abstractmethod
+    async def _legacy_crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]: ...

{unique_web_search-2026.30.0.dev2 → unique_web_search-2026.30.0.dev3}/src/unique_web_search/services/crawlers/basic.py

@@ -1,84 +1,82 @@
 import asyncio
 import logging
 import re
-from typing import Literal
+from typing import Annotated
 
 import timeout_decorator
 from httpx import AsyncClient, Timeout
 from markdownify import markdownify
 from pydantic import Field
 from typing_extensions import override
+from unique_search_proxy_core.crawlers.base import CrawlerType
+from unique_search_proxy_core.crawlers.basic.content_types import (
+    CONTENT_TYPE_TOGGLE_TO_MIME,
+)
+from unique_search_proxy_core.crawlers.basic.schema import (
+    BasicConfig as CoreBasicConfig,
+)
+from unique_toolkit._common.pydantic.rjsf_tags import RJSFMetaTag
 
 from unique_web_search.services.client.proxy_config import async_client
-from unique_web_search.services.crawlers.base import (
-    BaseCrawler,
-    BaseCrawlerConfig,
-    CrawlerType,
-)
+from unique_web_search.services.crawlers.base import BaseCrawler
+from unique_web_search.services.crawlers.registry import register_crawler
 from unique_web_search.services.crawlers.url_safety import (
     CrawlTargetValidationError,
     ResolvedCrawlTarget,
 )
 from unique_web_search.services.crawlers.utils import get_random_user_agent
-from unique_web_search.services.proxy.bridge import (
-    open_search_proxy_client,
-)
+from unique_web_search.services.proxy.bridge import open_search_proxy_client
 from unique_web_search.services.proxy.mappers import map_crawl_response
 
 _LOGGER = logging.getLogger(__name__)
 
-unwanted_types = {
-    "application/octet-stream",
-    "application/pdf",
-    "application/msword",
-    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-    "image/",
-    "video/",
-    "audio/",
-}
-
-
-class BasicCrawlerConfig(BaseCrawlerConfig[CrawlerType.BASIC]):
-    crawler_type: Literal[CrawlerType.BASIC] = CrawlerType.BASIC
-    url_pattern_blacklist: list[str] = Field(
-        default=[r".*\.pdf$"],
-        description="List of URL patterns to blacklist",
-    )
-    unwanted_content_types: set[str] = Field(
-        default=unwanted_types,
-        description="Set of content types to not allow",
-    )
-    max_concurrent_requests: int = Field(
-        default=10,
-        description="The maximum number of concurrent requests to make to the same domain.",
-    )
 
+class BasicConfig(CoreBasicConfig):
+    """Tool-local Basic crawler config; extends proxy-core with legacy URL filters."""
+
+    url_blocked_patterns: Annotated[
+        list[str],
+        RJSFMetaTag({"ui:options": {"orderable": False}}),
+    ] = Field(
+        default_factory=lambda: [r".*\.pdf$"],
+        title="URL blocked patterns",
+        description=(
+            "List of URL regex patterns to skip when crawling. "
+            "**Note:** This field will be deprecated in the future."
+        ),
+    )
 
-class BasicCrawler(BaseCrawler[BasicCrawlerConfig]):
-    supports_proxy_crawl = True
 
-    def __init__(self, config: BasicCrawlerConfig):
+@register_crawler(
+    name="basic",
+    key=CrawlerType.BASIC,
+    config_cls=BasicConfig,
+    config_display_name="Basic",
+)
+class BasicCrawler(BaseCrawler[BasicConfig]):
+    def __init__(self, config: BasicConfig):
         super().__init__(config)
 
         self.semaphore: asyncio.Semaphore = asyncio.Semaphore(
             self.config.max_concurrent_requests
         )
 
-    # TODO: Find a good way for tracking
-    # @track(
-    #     tags=["basic", "scrape"],
-    # )
     @override
     async def _proxy_crawl(self, urls: list[str]) -> list[str]:
+        """Proxy crawl excludes tool-only fields the search-proxy schema forbids."""
+        params = self.config.model_dump(
+            exclude={"crawler", "url_blocked_patterns"},
+            exclude_none=True,
+        )
         async with open_search_proxy_client(
-            timeout=float(self.config.timeout),
+            timeout=float(self.config.timeout)
         ) as client:
-            response = await client.crawl.basic(
+            response = await client.crawl.crawl(
                 urls=urls,
-                timeout=int(self.config.timeout),
-                max_concurrent_requests=self.config.max_concurrent_requests,
+                crawler=self.config.crawler,
+                **params,
             )
-            return map_crawl_response(response, urls)
+        return map_crawl_response(response, urls)
 
     @override
     async def _legacy_crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]:
@@ -111,8 +109,8 @@ class BasicCrawler(BaseCrawler[BasicCrawlerConfig]):
     ) -> str:
         headers = {"User-Agent": get_random_user_agent()}
 
-        if self._is_url_blacklisted(target.normalized_url):
-            return "Unable to crawl URL due to blacklisted pattern"
+        if self._is_url_blocked(target.normalized_url):
+            return "Unable to crawl URL due to blocked pattern"
 
         request_headers = dict(headers)
         request_extensions: dict[str, str] = {}
@@ -142,13 +140,18 @@ class BasicCrawler(BaseCrawler[BasicCrawlerConfig]):
 
         return markdown
 
-    def _is_url_blacklisted(self, url: str) -> bool:
+    def _is_url_blocked(self, url: str) -> bool:
         return any(
-            re.match(pattern, url) for pattern in self.config.url_pattern_blacklist
+            re.match(pattern, url) for pattern in self.config.url_blocked_patterns
         )
 
     def _content_type_not_allowed(self, content_type: str) -> bool:
-        return content_type in self.config.unwanted_content_types
+        allowed_mimes = {
+            mime
+            for field_name, mime in CONTENT_TYPE_TOGGLE_TO_MIME.items()
+            if getattr(self.config.content_types, field_name)
+        }
+        return content_type not in allowed_mimes
 
 
 def _markdownify_html_with_timeout(content: str, timeout: float) -> str:

unique_web_search-2026.30.0.dev3/src/unique_web_search/services/crawlers/firecrawl.py

@@ -0,0 +1,49 @@
+from typing import override
+
+from firecrawl import AsyncFirecrawl
+from unique_search_proxy_core.crawlers.base import CrawlerType
+from unique_search_proxy_core.crawlers.firecrawl.schema import FirecrawlConfig
+
+from unique_web_search.client_settings import (
+    get_firecrawl_search_settings,
+)
+from unique_web_search.services.crawlers.base import BaseCrawler
+from unique_web_search.services.crawlers.registry import register_crawler
+from unique_web_search.services.crawlers.url_safety import ResolvedCrawlTarget
+
+
+@register_crawler(
+    name="firecrawl",
+    key=CrawlerType.FIRECRAWL,
+    config_cls=FirecrawlConfig,
+    config_display_name="Firecrawl",
+)
+class FirecrawlCrawler(BaseCrawler[FirecrawlConfig]):
+    def __init__(self, config: FirecrawlConfig):
+        super().__init__(config)
+
+    @override
+    async def _legacy_crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]:
+        urls = [target.normalized_url for target in targets]
+
+        api_key = get_firecrawl_search_settings().api_key
+        assert api_key is not None, "Firecrawl API key is not configured"
+
+        client = AsyncFirecrawl(api_key=api_key)
+
+        response = await client.batch_scrape(
+            urls=urls,
+            formats=["markdown"],
+            wait_timeout=self.config.timeout,
+        )
+
+        results = response.data
+
+        markdowns = [
+            result.markdown
+            if result.markdown is not None
+            else "An error occurred while scraping the URL"
+            for result in results
+        ]
+
+        return markdowns

{unique_web_search-2026.30.0.dev2 → unique_web_search-2026.30.0.dev3}/src/unique_web_search/services/crawlers/jina.py

@@ -1,21 +1,15 @@
 import asyncio
-from typing import Any, Literal, cast, override
+from typing import Any, override
 
 from httpx import AsyncClient
 from pydantic import BaseModel, Field, HttpUrl
-from unique_search_proxy_core.crawlers.jina.schema import JinaEngine, JinaReturnFormat
+from unique_search_proxy_core.crawlers.base import CrawlerType
+from unique_search_proxy_core.crawlers.jina.schema import JinaConfig
 
 from unique_web_search.client_settings import get_jina_search_settings
-from unique_web_search.services.crawlers.base import (
-    BaseCrawler,
-    BaseCrawlerConfigExperimental,
-    CrawlerType,
-)
+from unique_web_search.services.crawlers.base import BaseCrawler
+from unique_web_search.services.crawlers.registry import register_crawler
 from unique_web_search.services.crawlers.url_safety import ResolvedCrawlTarget
-from unique_web_search.services.proxy.bridge import (
-    open_search_proxy_client,
-)
-from unique_web_search.services.proxy.mappers import map_crawl_response
 
 
 class ReaderBody(BaseModel):
@@ -38,46 +32,16 @@ class ReaderResponse(BaseModel):
     data: ReaderData | None = None
 
 
-class JinaCrawlerConfig(BaseCrawlerConfigExperimental[CrawlerType.JINA]):
-    crawler_type: Literal[CrawlerType.JINA] = CrawlerType.JINA
-    headers: dict[str, str] = Field(
-        default={
-            "X-Return-Format": "markdown",
-            "X-Engine": "browser",
-            "DNT": "1",
-        },
-        description="Headers to send with the request",
-    )
-
-
-class JinaCrawler(BaseCrawler[JinaCrawlerConfig]):
-    supports_proxy_crawl = True
-
-    def __init__(self, config: JinaCrawlerConfig):
+@register_crawler(
+    name="jina",
+    key=CrawlerType.JINA,
+    config_cls=JinaConfig,
+    config_display_name="Jina",
+)
+class JinaCrawler(BaseCrawler[JinaConfig]):
+    def __init__(self, config: JinaConfig):
         super().__init__(config)
 
-    # TODO: Find a solution for tracking
-    # @track(
-    #     tags=["jina", "scrape"],
-    # )
-    @override
-    async def _proxy_crawl(self, urls: list[str]) -> list[str]:
-        headers = self.config.headers
-        async with open_search_proxy_client(
-            timeout=float(self.config.timeout),
-        ) as client:
-            response = await client.crawl.jina(
-                urls=urls,
-                timeout=int(self.config.timeout),
-                return_format=cast(
-                    JinaReturnFormat,
-                    headers.get("X-Return-Format", "markdown"),
-                ),
-                engine=cast(JinaEngine, headers.get("X-Engine", "browser")),
-                do_not_track=headers.get("DNT") == "1",
-            )
-            return map_crawl_response(response, urls)
-
     @override
     async def _legacy_crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]:
         urls = [target.normalized_url for target in targets]
@@ -91,7 +55,7 @@ class JinaCrawler(BaseCrawler[JinaCrawlerConfig]):
             "Authorization": f"Bearer {api_key}",
             "Content-Type": "application/json",
             "Accept": "application/json",
-        } | self.config.headers
+        }
 
         async with AsyncClient(timeout=self.config.timeout) as client:
             tasks = [
@@ -116,6 +80,52 @@ class JinaCrawler(BaseCrawler[JinaCrawlerConfig]):
 
         return markdown_results
 
+    def _build_reader_body(self, url: str) -> dict[str, Any]:
+        """Build the Jina Reader POST body applying all deployment config fields.
+
+        Mirrors ``build_jina_reader_body`` on the search-proxy path so the direct
+        (proxy-disabled) crawl honours the same ``JinaConfig`` fields.
+        """
+        config = self.config
+        page_timeout = config.page_timeout
+        if page_timeout is None:
+            page_timeout = min(max(config.timeout, 1), 180)
+
+        body: dict[str, Any] = {
+            "url": url,
+            "respondWith": config.return_format,
+            "engine": config.engine,
+            "timeout": page_timeout,
+            "doNotTrack": config.do_not_track,
+        }
+
+        if config.no_cache:
+            body["noCache"] = True
+        if config.target_selector is not None:
+            body["targetSelector"] = config.target_selector
+        if config.wait_for_selector is not None:
+            body["waitForSelector"] = config.wait_for_selector
+        if config.remove_selector is not None:
+            body["removeSelector"] = config.remove_selector
+        if config.with_generated_alt:
+            body["withGeneratedAlt"] = True
+        if config.with_links_summary:
+            body["withLinksSummary"] = True
+        if config.with_images_summary:
+            body["withImagesSummary"] = True
+        if config.with_iframe:
+            body["withIframe"] = True
+        if config.retain_images is not None:
+            body["retainImages"] = config.retain_images
+        if config.locale is not None:
+            body["locale"] = config.locale
+        if config.referer is not None:
+            body["referer"] = config.referer
+        if config.proxy_url is not None:
+            body["proxyUrl"] = config.proxy_url
+
+        return body
+
     async def _crawl_url(
         self,
         url: str,
@@ -126,7 +136,7 @@ class JinaCrawler(BaseCrawler[JinaCrawlerConfig]):
         params = {
             "url": reader_api_endpoint,
             "headers": headers,
-            "json": {"url": url},
+            "json": self._build_reader_body(url),
         }
         response = await client.post(**params)
         return ReaderResponse.model_validate(response.json())