firecrawl-haystack 1.0.1__tar.gz → 1.2.0__tar.gz

firecrawl_haystack-1.2.0/CHANGELOG.md

@@ -0,0 +1,22 @@
+# Changelog
+
+## [integrations/firecrawl-v1.1.0] - 2026-03-09
+
+### 🚀 Features
+
+- Add FirecrawlWebSearch component (#2888)
+
+
+## [integrations/firecrawl-v1.0.1] - 2026-02-24
+
+### 🌀 Miscellaneous
+
+- Docs: Update Firecrawl module path in Docusaurus configuration (#2869)
+
+## [integrations/firecrawl-v1.0.0] - 2026-02-24
+
+### 🚀 Features
+
+- Add FirecrawlCrawler (#2859)
+
+<!-- generated by git-cliff -->

firecrawl_haystack-1.0.1/LICENSE → firecrawl_haystack-1.2.0/LICENSE.txt

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2023 deepset GmbH
+   Copyright 2023-present deepset GmbH
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

{firecrawl_haystack-1.0.1 → firecrawl_haystack-1.2.0}/PKG-INFO

@@ -1,14 +1,14 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: firecrawl-haystack
-Version: 1.0.1
+Version: 1.2.0
 Summary: Haystack integration for Firecrawl
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/firecrawl#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/firecrawl
 Author-email: deepset GmbH <info@deepset.ai>
 License-Expression: Apache-2.0
-License-File: LICENSE
-Keywords: Firecrawl,Haystack,Web Crawler,Web Scraping
+License-File: LICENSE.txt
+Keywords: Firecrawl,Haystack,Web Crawler,Web Scraping,Web Search
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Requires-Python: >=3.10
 Requires-Dist: firecrawl-py>=4.0.0
@@ -27,6 +28,7 @@ Description-Content-Type: text/markdown
 [![PyPI - Version](https://img.shields.io/pypi/v/firecrawl-haystack.svg)](https://pypi.org/project/firecrawl-haystack)
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/firecrawl-haystack.svg)](https://pypi.org/project/firecrawl-haystack)
 
+- [Integration page](https://haystack.deepset.ai/integrations/firecrawl)
 - [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/firecrawl/CHANGELOG.md)
 
 ---
@@ -35,5 +37,4 @@ Description-Content-Type: text/markdown
 
 Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
 
-
-To run integration tests locally, you need to export the `FIRECRAWL_API_KEY` environment variable. 
+To run integration tests locally, you need to export the `FIRECRAWL_API_KEY` environment variable. 

{firecrawl_haystack-1.0.1 → firecrawl_haystack-1.2.0}/README.md

@@ -3,6 +3,7 @@
 [![PyPI - Version](https://img.shields.io/pypi/v/firecrawl-haystack.svg)](https://pypi.org/project/firecrawl-haystack)
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/firecrawl-haystack.svg)](https://pypi.org/project/firecrawl-haystack)
 
+- [Integration page](https://haystack.deepset.ai/integrations/firecrawl)
 - [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/firecrawl/CHANGELOG.md)
 
 ---
@@ -11,5 +12,4 @@
 
 Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
 
-
-To run integration tests locally, you need to export the `FIRECRAWL_API_KEY` environment variable. 
+To run integration tests locally, you need to export the `FIRECRAWL_API_KEY` environment variable. 

{firecrawl_haystack-1.0.1 → firecrawl_haystack-1.2.0}/pydoc/config_docusaurus.yml

@@ -1,6 +1,7 @@
 loaders:
   - modules:
       - haystack_integrations.components.fetchers.firecrawl.firecrawl_crawler
+      - haystack_integrations.components.websearch.firecrawl.firecrawl_websearch
     search_path: [../src]
 processors:
   - type: filter

{firecrawl_haystack-1.0.1 → firecrawl_haystack-1.2.0}/pyproject.toml

@@ -14,6 +14,7 @@ keywords = [
     "Haystack",
     "Web Crawler",
     "Web Scraping",
+    "Web Search",
 ]
 authors = [
     { name = "deepset GmbH", email = "info@deepset.ai" },
@@ -26,6 +27,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Programming Language :: Python :: Implementation :: CPython",
 ]
 dependencies = [
@@ -72,8 +74,9 @@ dependencies = [
 unit = 'pytest -m "not integration" {args:tests}'
 integration = 'pytest -m "integration" {args:tests}'
 all = 'pytest {args:tests}'
-cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
-types = "mypy -p haystack_integrations.components.fetchers.firecrawl {args}"
+unit-cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x -m "not integration" {args:tests}'
+integration-cov-append-retry = 'pytest --cov=haystack_integrations --cov-append --reruns 3 --reruns-delay 30 -x -m "integration" {args:tests}'
+types = "mypy -p haystack_integrations.components.fetchers.firecrawl -p haystack_integrations.components.websearch.firecrawl {args}"
 
 [tool.mypy]
 install_types = true
@@ -87,9 +90,17 @@ line-length = 120
 [tool.ruff.lint]
 select = [
     "A",
+    "ANN",
     "ARG",
     "B",
     "C",
+    "D102",   # Missing docstring in public method
+    "D103",   # Missing docstring in public function
+    "D205",   # 1 blank line required between summary line and description
+    "D209",   # Closing triple quotes go to new line
+    "D213",   # summary lines must be positioned on the second physical line of the docstring
+    "D417",   # Missing argument descriptions in the docstring
+    "D419",   # Docstring is empty
     "DTZ",
     "E",
     "EM",
@@ -122,12 +133,15 @@ ignore = [
     "S105",
     "S106",
     "S107",
+    # Allow `Any` - used legitimately for dynamic types and SDK boundaries
+    "ANN401",
     # Ignore complexity
     "C901",
     "PLR0911",
     "PLR0912",
     "PLR0913",
     "PLR0915",
+    "PLR0917",
 ]
 unfixable = [
     # Don't touch unused imports
@@ -142,11 +156,12 @@ ban-relative-imports = "parents"
 
 [tool.ruff.lint.per-file-ignores]
 # Tests can use magic values, assertions, and relative imports
-"tests/**/*" = ["PLR2004", "S101", "TID252"]
+"tests/**/*" = ["D", "PLR2004", "S101", "TID252", "ANN"]
 
 [tool.coverage.run]
 source = ["haystack_integrations"]
 branch = true
+relative_files = true
 parallel = false
 
 [tool.coverage.report]

{firecrawl_haystack-1.0.1 → firecrawl_haystack-1.2.0}/src/haystack_integrations/components/fetchers/firecrawl/firecrawl_crawler.py

@@ -27,15 +27,15 @@ class FirecrawlCrawler:
     ### Usage example
 
     ```python
-    from haystack_integrations.components.fetchers.firecrawl import FirecrawlFetcher
+    from haystack_integrations.components.fetchers.firecrawl import FirecrawlCrawler
 
-    fetcher = FirecrawlFetcher(
+    crawler = FirecrawlCrawler(
         api_key=Secret.from_env_var("FIRECRAWL_API_KEY"),
         params={"limit": 5},
     )
-    fetcher.warm_up()
+    crawler.warm_up()
 
-    result = fetcher.run(urls=["https://docs.haystack.deepset.ai/docs/intro"])
+    result = crawler.run(urls=["https://docs.haystack.deepset.ai/docs/intro"])
     documents = result["documents"]
     ```
     """
@@ -46,7 +46,7 @@ class FirecrawlCrawler:
         params: dict[str, Any] | None = None,
     ) -> None:
         """
-        Initialize the FirecrawlFetcher.
+        Initialize the FirecrawlCrawler.
 
         :param api_key:
             API key for Firecrawl.
@@ -83,14 +83,18 @@ class FirecrawlCrawler:
         :returns: A dictionary with the following keys:
             - `documents`: List of documents, one for each URL crawled.
         """
-        if self._firecrawl_client is None:
-            self.warm_up()
+        self.warm_up()
+        assert self._firecrawl_client is not None  # noqa: S101
 
         current_params = params if params is not None else self._params
         documents: list[Document] = []
         for url in urls:
-            docs = self._crawl_url(url=url, params=current_params)
-            documents.extend(docs)
+            try:
+                crawl_response = self._firecrawl_client.crawl(url=url, **current_params)
+            except Exception as error:
+                logger.exception(f"Failed to crawl website {url}: {error}")
+                continue
+            documents.extend(self._documents_from_crawl_response(url=url, crawl_response=crawl_response))
 
         return {"documents": documents}
 
@@ -111,67 +115,31 @@ class FirecrawlCrawler:
         :returns: A dictionary with the following keys:
             - `documents`: List of documents, one for each URL crawled.
         """
-        if self._async_firecrawl_client is None:
-            self.warm_up()
+        await self.warm_up_async()
+        assert self._async_firecrawl_client is not None  # noqa: S101
 
         current_params = params if params is not None else self._params
         documents: list[Document] = []
         for url in urls:
-            docs = await self._crawl_url_async(url=url, params=current_params)
-            documents.extend(docs)
+            try:
+                crawl_response = await self._async_firecrawl_client.crawl(url=url, **current_params)
+            except Exception as error:
+                logger.exception(f"Failed to crawl website {url}: {error}")
+                continue
+            documents.extend(self._documents_from_crawl_response(url=url, crawl_response=crawl_response))
 
         return {"documents": documents}
 
     def warm_up(self) -> None:
-        """
-        Warm up the Firecrawl client by initializing the clients.
-        This is useful to avoid cold start delays when crawling many URLs.
-        """
+        """Warm up the synchronous Firecrawl client."""
         if self._firecrawl_client is None:
             self._firecrawl_client = Firecrawl(api_key=self.api_key.resolve_value())
+
+    async def warm_up_async(self) -> None:
+        """Warm up the asynchronous Firecrawl client."""
         if self._async_firecrawl_client is None:
             self._async_firecrawl_client = AsyncFirecrawl(api_key=self.api_key.resolve_value())
 
-    def _crawl_url(self, url: str, params: dict[str, Any]) -> list[Document]:
-        """
-        Crawl a single URL and return Documents.
-
-        :param url: URL to crawl.
-        :param params: Crawl request parameters.
-        :return: List of Documents from the crawl result.
-        """
-        try:
-            # Ignoring type because the firecrawl client is initialized as None and only set during warm_up
-            crawl_response = self._firecrawl_client.crawl(  # type: ignore[union-attr]
-                url=url,
-                **params,
-            )
-        except Exception as error:
-            logger.exception(f"Failed to crawl website {url}: {error}")
-            return []
-
-        return self._documents_from_crawl_response(url=url, crawl_response=crawl_response)
-
-    async def _crawl_url_async(self, url: str, params: dict[str, Any]) -> list[Document]:
-        """
-        Asynchronously crawl a single URL and return Documents.
-
-        :param url: URL to crawl.
-        :param params: Crawl request parameters.
-        :return: List of Documents from the crawl result.
-        """
-        try:
-            # Ignoring type because the firecrawl client is initialized as None and only set during warm_up
-            crawl_response = await self._async_firecrawl_client.crawl(  # type: ignore[union-attr]
-                url=url,
-                **params,
-            )
-        except Exception as error:
-            logger.exception(f"Failed to crawl website {url}: {error}")
-            return []
-
-        return self._documents_from_crawl_response(url=url, crawl_response=crawl_response)
-
     def _documents_from_crawl_response(self, url: str, crawl_response: Any) -> list[Document]:
         """
         Convert a Firecrawl crawl response to Haystack Documents.

firecrawl_haystack-1.2.0/src/haystack_integrations/components/websearch/firecrawl/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2026-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from haystack_integrations.components.websearch.firecrawl.firecrawl_websearch import FirecrawlWebSearch
+
+__all__ = ["FirecrawlWebSearch"]

firecrawl_haystack-1.2.0/src/haystack_integrations/components/websearch/firecrawl/firecrawl_websearch.py

@@ -0,0 +1,194 @@
+# SPDX-FileCopyrightText: 2026-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack import Document, component, logging
+from haystack.utils import Secret
+
+from firecrawl import AsyncFirecrawl, Firecrawl  # type: ignore[import-untyped]
+from firecrawl.types import SearchData  # type: ignore[import-untyped]
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class FirecrawlWebSearch:
+    """
+    A component that uses Firecrawl to search the web and return results as Haystack Documents.
+
+    This component wraps the Firecrawl Search API, enabling web search queries that return
+    structured documents with content and links. It follows the standard Haystack WebSearch
+    component interface.
+
+    Firecrawl is a service that crawls and scrapes websites, returning content in formats suitable
+    for LLMs. You need a Firecrawl API key from [firecrawl.dev](https://firecrawl.dev).
+
+    ### Usage example
+
+    ```python
+    from haystack_integrations.components.websearch.firecrawl import FirecrawlWebSearch
+    from haystack.utils import Secret
+
+    websearch = FirecrawlWebSearch(
+        api_key=Secret.from_env_var("FIRECRAWL_API_KEY"),
+        top_k=5,
+    )
+    result = websearch.run(query="What is Haystack by deepset?")
+    documents = result["documents"]
+    links = result["links"]
+    ```
+    """
+
+    def __init__(
+        self,
+        api_key: Secret = Secret.from_env_var("FIRECRAWL_API_KEY"),
+        top_k: int | None = 10,
+        search_params: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Initialize the FirecrawlWebSearch component.
+
+        :param api_key:
+            API key for Firecrawl.
+            Defaults to the `FIRECRAWL_API_KEY` environment variable.
+        :param top_k:
+            Maximum number of documents to return.
+            Defaults to 10. This can be overridden by the `"limit"` parameter in `search_params`.
+        :param search_params:
+            Additional parameters passed to the Firecrawl search API.
+            See the [Firecrawl API reference](https://docs.firecrawl.dev/api-reference/endpoint/search)
+            for available parameters. Supported keys include: `tbs`, `location`,
+            `scrape_options`, `sources`, `categories`, `timeout`.
+        """
+        self.api_key = api_key
+        self.top_k = top_k
+        self.search_params = search_params
+        self._search_params = {} if search_params is None else search_params.copy()
+        self._firecrawl_client: Firecrawl | None = None
+        self._async_firecrawl_client: AsyncFirecrawl | None = None
+
+    def warm_up(self) -> None:
+        """Warm up the synchronous Firecrawl client."""
+        if self._firecrawl_client is None:
+            self._firecrawl_client = Firecrawl(api_key=self.api_key.resolve_value())
+
+    async def warm_up_async(self) -> None:
+        """Warm up the asynchronous Firecrawl client."""
+        if self._async_firecrawl_client is None:
+            self._async_firecrawl_client = AsyncFirecrawl(api_key=self.api_key.resolve_value())
+
+    @component.output_types(documents=list[Document], links=list[str])
+    def run(
+        self,
+        query: str,
+        search_params: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Search the web using Firecrawl and return results as Documents.
+
+        :param query: Search query string.
+        :param search_params:
+            Optional override of search parameters for this run.
+            If provided, fully replaces the init-time search_params.
+        :returns: A dictionary with the following keys:
+            - `documents`: List of documents with search result content.
+            - `links`: List of URLs from the search results.
+        """
+        self.warm_up()
+        assert self._firecrawl_client is not None  # noqa: S101
+
+        current_params = search_params if search_params is not None else self._search_params
+        params = current_params.copy()
+        if "limit" not in params and self.top_k is not None:
+            params["limit"] = self.top_k
+
+        try:
+            search_response = self._firecrawl_client.search(
+                query=query,
+                **params,
+            )
+        except Exception as error:
+            logger.exception(f"Failed to search for query '{query}': {error}")
+            return {"documents": [], "links": []}
+
+        documents, links = self._parse_search_response(search_response)
+        return {"documents": documents, "links": links}
+
+    @component.output_types(documents=list[Document], links=list[str])
+    async def run_async(
+        self,
+        query: str,
+        search_params: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Asynchronously search the web using Firecrawl and return results as Documents.
+
+        :param query: Search query string.
+        :param search_params:
+            Optional override of search parameters for this run.
+            If provided, fully replaces the init-time search_params.
+        :returns: A dictionary with the following keys:
+            - `documents`: List of documents with search result content.
+            - `links`: List of URLs from the search results.
+        """
+        await self.warm_up_async()
+        assert self._async_firecrawl_client is not None  # noqa: S101
+
+        current_params = search_params if search_params is not None else self._search_params
+        params = current_params.copy()
+        if "limit" not in params and self.top_k is not None:
+            params["limit"] = self.top_k
+
+        try:
+            search_response = await self._async_firecrawl_client.search(
+                query=query,
+                **params,
+            )
+        except Exception as error:
+            logger.exception(f"Failed to search for query '{query}': {error}")
+            return {"documents": [], "links": []}
+
+        documents, links = self._parse_search_response(search_response)
+        return {"documents": documents, "links": links}
+
+    @staticmethod
+    def _parse_search_response(search_response: SearchData) -> tuple[list[Document], list[str]]:
+        """
+        Convert a Firecrawl search response to Haystack Documents and links.
+
+        :param search_response: Firecrawl search response object.
+        :returns: Tuple of (documents, links).
+        """
+        documents: list[Document] = []
+        links: list[str] = []
+
+        web_results = search_response.web or []
+        for result in web_results:
+            url = ""
+            title = ""
+            content = ""
+
+            if hasattr(result, "markdown") and result.markdown:
+                content = result.markdown
+                metadata = result.metadata_dict if hasattr(result, "metadata_dict") else {}
+                url = metadata.get("url", getattr(result, "url", ""))
+                title = metadata.get("title", "")
+            else:
+                url = getattr(result, "url", "")
+                title = getattr(result, "title", "")
+                content = getattr(result, "description", "")
+
+            doc = Document(
+                content=content,
+                meta={
+                    "title": title,
+                    "url": url,
+                },
+            )
+            documents.append(doc)
+            if url:
+                links.append(url)
+
+        return documents, links

{firecrawl_haystack-1.0.1 → firecrawl_haystack-1.2.0}/tests/test_firecrawl_crawler.py

@@ -14,38 +14,44 @@ from haystack.utils import Secret
 from haystack_integrations.components.fetchers.firecrawl import FirecrawlCrawler
 
 
-class TestFirecrawlCrawler:
-    @pytest.fixture
-    def page(self) -> MagicMock:
-        page = MagicMock()
-        page.markdown = "Page content"
-        page.metadata_dict = {"url": "https://example.com"}
-        return page
-
-    @pytest.fixture
-    def crawl_response(self, page) -> MagicMock:
-        crawl_response = MagicMock()
-        crawl_response.status = "completed"
-        crawl_response.data = [page]
-        return crawl_response
-
-    @pytest.fixture
-    def mock_client(self, crawl_response) -> MagicMock:
-        mock_client = MagicMock()
-        mock_client.crawl.return_value = crawl_response
-        return mock_client
-
-    @pytest.fixture
-    def mock_async_client(self, crawl_response) -> MagicMock:
-        mock_async_client = MagicMock()
-        mock_async_client.crawl = AsyncMock(return_value=crawl_response)
-        return mock_async_client
+@pytest.fixture
+def page() -> MagicMock:
+    page = MagicMock()
+    page.markdown = "Page content"
+    page.metadata_dict = {"url": "https://example.com"}
+    return page
 
+
+@pytest.fixture
+def crawl_response(page) -> MagicMock:
+    crawl_response = MagicMock()
+    crawl_response.status = "completed"
+    crawl_response.data = [page]
+    return crawl_response
+
+
+@pytest.fixture
+def mock_client(crawl_response) -> MagicMock:
+    mock_client = MagicMock()
+    mock_client.crawl.return_value = crawl_response
+    return mock_client
+
+
+@pytest.fixture
+def mock_async_client(crawl_response) -> MagicMock:
+    mock_async_client = MagicMock()
+    mock_async_client.crawl = AsyncMock(return_value=crawl_response)
+    return mock_async_client
+
+
+class TestInit:
     def test_init_default(self, monkeypatch: pytest.MonkeyPatch) -> None:
         monkeypatch.setenv("FIRECRAWL_API_KEY", "test-key")
         fetcher = FirecrawlCrawler()
         assert fetcher._params == {"limit": 1, "scrape_options": {"formats": ["markdown"]}}
         assert fetcher.api_key.resolve_value() == "test-key"
+        assert fetcher._firecrawl_client is None
+        assert fetcher._async_firecrawl_client is None
 
     def test_init_with_params(self) -> None:
         fetcher = FirecrawlCrawler(
@@ -55,13 +61,14 @@ class TestFirecrawlCrawler:
         assert fetcher._params == {"limit": 10, "scrape_options": {"formats": ["markdown"]}}
         assert fetcher.api_key.resolve_value() == "custom-key"
 
-    def test_to_dict(self, monkeypatch: pytest.MonkeyPatch) -> None:
-        monkeypatch.setenv("FIRECRAWL_API_KEY", "test-key")
+
+class TestSerialization:
+    def test_to_dict(self) -> None:
         fetcher = FirecrawlCrawler(
             api_key=Secret.from_env_var("FIRECRAWL_API_KEY"),
             params={"limit": 5},
         )
-        data = component_to_dict(fetcher, "FirecrawlFetcher")
+        data = component_to_dict(fetcher, "FirecrawlCrawler")
         assert data["type"] == "haystack_integrations.components.fetchers.firecrawl.firecrawl_crawler.FirecrawlCrawler"
         assert data["init_parameters"]["params"] == {"limit": 5}
         assert "api_key" in data["init_parameters"]
@@ -75,10 +82,46 @@ class TestFirecrawlCrawler:
                 "api_key": {"env_vars": ["FIRECRAWL_API_KEY"], "strict": True, "type": "env_var"},
             },
         }
-        fetcher = component_from_dict(FirecrawlCrawler, data, "FirecrawlFetcher")
+        fetcher = component_from_dict(FirecrawlCrawler, data, "FirecrawlCrawler")
         assert fetcher.params == {"limit": 3}
         assert fetcher.api_key.resolve_value() == "test-key"
 
+
+class TestComponentLifecycle:
+    def test_key_resolved_at_warm_up_not_init(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False)
+        fetcher = FirecrawlCrawler()
+
+        with pytest.raises(ValueError, match="None of the following authentication environment variables are set"):
+            fetcher.warm_up()
+
+    def test_warm_up(self) -> None:
+        fetcher = FirecrawlCrawler(api_key=Secret.from_token("test-key"))
+
+        with patch("haystack_integrations.components.fetchers.firecrawl.firecrawl_crawler.Firecrawl") as mock_client:
+            fetcher.warm_up()
+            fetcher.warm_up()
+
+        assert fetcher._firecrawl_client is mock_client.return_value
+        assert fetcher._async_firecrawl_client is None
+        mock_client.assert_called_once_with(api_key="test-key")
+
+    @pytest.mark.asyncio
+    async def test_warm_up_async(self) -> None:
+        fetcher = FirecrawlCrawler(api_key=Secret.from_token("test-key"))
+
+        with patch(
+            "haystack_integrations.components.fetchers.firecrawl.firecrawl_crawler.AsyncFirecrawl"
+        ) as mock_async_client:
+            await fetcher.warm_up_async()
+            await fetcher.warm_up_async()
+
+        assert fetcher._firecrawl_client is None
+        assert fetcher._async_firecrawl_client is mock_async_client.return_value
+        mock_async_client.assert_called_once_with(api_key="test-key")
+
+
+class TestRun:
     def test_run_with_mocked_firecrawl_client(self, mock_client) -> None:
         fetcher = FirecrawlCrawler(api_key=Secret.from_token("test-key"))
 
@@ -135,12 +178,9 @@ class TestFirecrawlCrawler:
         )
 
     def test_run_calls_warm_up(self, crawl_response) -> None:
-        with (
-            patch(
-                "haystack_integrations.components.fetchers.firecrawl.firecrawl_crawler.Firecrawl"
-            ) as mock_firecrawl_client,
-            patch("haystack_integrations.components.fetchers.firecrawl.firecrawl_crawler.AsyncFirecrawl"),
-        ):
+        with patch(
+            "haystack_integrations.components.fetchers.firecrawl.firecrawl_crawler.Firecrawl"
+        ) as mock_firecrawl_client:
             mock_firecrawl_client.return_value.crawl.return_value = crawl_response
 
             fetcher = FirecrawlCrawler(api_key=Secret.from_token("test-key"))
@@ -151,12 +191,9 @@ class TestFirecrawlCrawler:
 
     @pytest.mark.asyncio
     async def test_run_async_calls_warm_up(self, crawl_response) -> None:
-        with (
-            patch("haystack_integrations.components.fetchers.firecrawl.firecrawl_crawler.Firecrawl"),
-            patch(
-                "haystack_integrations.components.fetchers.firecrawl.firecrawl_crawler.AsyncFirecrawl"
-            ) as mock_async_firecrawl_client,
-        ):
+        with patch(
+            "haystack_integrations.components.fetchers.firecrawl.firecrawl_crawler.AsyncFirecrawl"
+        ) as mock_async_firecrawl_client:
             mock_async_firecrawl_client.return_value.crawl = AsyncMock(return_value=crawl_response)
 
             fetcher = FirecrawlCrawler(api_key=Secret.from_token("test-key"))
@@ -222,21 +259,13 @@ class TestFirecrawlCrawler:
 
         assert "Failed to crawl website https://example.com: failed" in caplog.text
 
-    def test_warm_up_initializes_clients(self) -> None:
-        fetcher = FirecrawlCrawler(api_key=Secret.from_token("test-key"))
-        assert fetcher._firecrawl_client is None
-        assert fetcher._async_firecrawl_client is None
-
-        fetcher.warm_up()
-
-        assert fetcher._firecrawl_client is not None
-        assert fetcher._async_firecrawl_client is not None
 
-    @pytest.mark.skipif(
-        not os.environ.get("FIRECRAWL_API_KEY"),
-        reason="Export FIRECRAWL_API_KEY to run integration tests.",
-    )
-    @pytest.mark.integration
+@pytest.mark.skipif(
+    not os.environ.get("FIRECRAWL_API_KEY"),
+    reason="Export FIRECRAWL_API_KEY to run integration tests.",
+)
+@pytest.mark.integration
+class TestIntegration:
     def test_run_integration(self) -> None:
         fetcher = FirecrawlCrawler(api_key=Secret.from_env_var("FIRECRAWL_API_KEY"), params={"limit": 1})
         result = fetcher.run(urls=["https://docs.haystack.deepset.ai/docs/intro"], params={"limit": 1})
@@ -247,11 +276,6 @@ class TestFirecrawlCrawler:
         assert "Haystack" in result["documents"][0].content
         assert "https://docs.haystack.deepset.ai/docs/intro" == result["documents"][0].meta["url"]
 
-    @pytest.mark.skipif(
-        not os.environ.get("FIRECRAWL_API_KEY"),
-        reason="Export FIRECRAWL_API_KEY to run integration tests.",
-    )
-    @pytest.mark.integration
     @pytest.mark.asyncio
     async def test_run_async_integration(self) -> None:
         fetcher = FirecrawlCrawler(api_key=Secret.from_env_var("FIRECRAWL_API_KEY"), params={"limit": 1})

firecrawl_haystack-1.2.0/tests/test_firecrawl_websearch.py

@@ -0,0 +1,301 @@
+# SPDX-FileCopyrightText: 2026-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from haystack import Document
+from haystack.core.serialization import component_from_dict, component_to_dict
+from haystack.utils import Secret
+
+from haystack_integrations.components.websearch.firecrawl import FirecrawlWebSearch
+
+
+@pytest.fixture
+def search_result_web() -> MagicMock:
+    """A SearchResultWeb-like object (no markdown, just search metadata)."""
+    result = MagicMock(spec=["url", "title", "description"])
+    result.url = "https://example.com"
+    result.title = "Example Title"
+    result.description = "Example description snippet"
+    return result
+
+
+@pytest.fixture
+def search_result_document() -> MagicMock:
+    """A Document-like object (with markdown, from scrapeOptions)."""
+    result = MagicMock(spec=["markdown", "metadata_dict", "url"])
+    result.markdown = "# Full page content\nSome markdown text."
+    result.metadata_dict = {"url": "https://example.com/page", "title": "Page Title"}
+    result.url = "https://example.com/page"
+    return result
+
+
+@pytest.fixture
+def search_response(search_result_web) -> MagicMock:
+    """Standard search response with web results."""
+    response = MagicMock()
+    response.web = [search_result_web]
+    return response
+
+
+@pytest.fixture
+def mock_client(search_response) -> MagicMock:
+    client = MagicMock()
+    client.search.return_value = search_response
+    return client
+
+
+@pytest.fixture
+def mock_async_client(search_response) -> MagicMock:
+    client = MagicMock()
+    client.search = AsyncMock(return_value=search_response)
+    return client
+
+
+class TestInit:
+    def test_init_default(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setenv("FIRECRAWL_API_KEY", "test-key")
+        ws = FirecrawlWebSearch()
+        assert ws.top_k == 10
+        assert ws._search_params == {}
+        assert ws.api_key.resolve_value() == "test-key"
+        assert ws._firecrawl_client is None
+        assert ws._async_firecrawl_client is None
+
+    def test_init_with_params(self) -> None:
+        ws = FirecrawlWebSearch(
+            api_key=Secret.from_token("custom-key"),
+            top_k=5,
+            search_params={"tbs": "qdr:d", "location": "US"},
+        )
+        assert ws.top_k == 5
+        assert ws._search_params == {"tbs": "qdr:d", "location": "US"}
+        assert ws.api_key.resolve_value() == "custom-key"
+
+
+class TestSerialization:
+    def test_to_dict(self) -> None:
+        ws = FirecrawlWebSearch(top_k=5, search_params={"tbs": "qdr:d"})
+        data = component_to_dict(ws, "FirecrawlWebSearch")
+        assert (
+            data["type"]
+            == "haystack_integrations.components.websearch.firecrawl.firecrawl_websearch.FirecrawlWebSearch"
+        )
+        assert data["init_parameters"]["top_k"] == 5
+        assert data["init_parameters"]["search_params"] == {"tbs": "qdr:d"}
+
+    def test_from_dict(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setenv("FIRECRAWL_API_KEY", "test-key")
+        data = {
+            "type": ("haystack_integrations.components.websearch.firecrawl.firecrawl_websearch.FirecrawlWebSearch"),
+            "init_parameters": {
+                "top_k": 3,
+                "search_params": {"location": "UK"},
+                "api_key": {"env_vars": ["FIRECRAWL_API_KEY"], "strict": True, "type": "env_var"},
+            },
+        }
+        ws = component_from_dict(FirecrawlWebSearch, data, "FirecrawlWebSearch")
+        assert ws.top_k == 3
+        assert ws.search_params == {"location": "UK"}
+        assert ws.api_key.resolve_value() == "test-key"
+
+
+class TestComponentLifecycle:
+    def test_key_resolved_at_warm_up_not_init(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False)
+        ws = FirecrawlWebSearch()
+
+        with pytest.raises(ValueError, match="None of the following authentication environment variables are set"):
+            ws.warm_up()
+
+    def test_warm_up(self) -> None:
+        ws = FirecrawlWebSearch(api_key=Secret.from_token("test-key"))
+
+        with patch("haystack_integrations.components.websearch.firecrawl.firecrawl_websearch.Firecrawl") as mock_client:
+            ws.warm_up()
+            ws.warm_up()
+
+        assert ws._firecrawl_client is mock_client.return_value
+        assert ws._async_firecrawl_client is None
+        mock_client.assert_called_once_with(api_key="test-key")
+
+    @pytest.mark.asyncio
+    async def test_warm_up_async(self) -> None:
+        ws = FirecrawlWebSearch(api_key=Secret.from_token("test-key"))
+
+        with patch(
+            "haystack_integrations.components.websearch.firecrawl.firecrawl_websearch.AsyncFirecrawl"
+        ) as mock_async_client:
+            await ws.warm_up_async()
+            await ws.warm_up_async()
+
+        assert ws._firecrawl_client is None
+        assert ws._async_firecrawl_client is mock_async_client.return_value
+        mock_async_client.assert_called_once_with(api_key="test-key")
+
+
+class TestRun:
+    def test_run_returns_documents_and_links(self, mock_client) -> None:
+        ws = FirecrawlWebSearch(api_key=Secret.from_token("test-key"), top_k=10)
+        ws._firecrawl_client = mock_client
+
+        result = ws.run(query="test query")
+
+        assert "documents" in result
+        assert "links" in result
+        assert len(result["documents"]) == 1
+        assert isinstance(result["documents"][0], Document)
+        assert result["documents"][0].content == "Example description snippet"
+        assert result["documents"][0].meta["url"] == "https://example.com"
+        assert result["documents"][0].meta["title"] == "Example Title"
+        assert result["links"] == ["https://example.com"]
+        mock_client.search.assert_called_once_with(query="test query", limit=10)
+
+    def test_run_with_scraped_documents(self, search_result_document, mock_client) -> None:
+        """When scrapeOptions are used, results contain markdown content."""
+        search_response = MagicMock()
+        search_response.web = [search_result_document]
+        mock_client.search.return_value = search_response
+
+        ws = FirecrawlWebSearch(api_key=Secret.from_token("test-key"), top_k=10)
+        ws._firecrawl_client = mock_client
+
+        result = ws.run(query="test query")
+
+        assert result["documents"][0].content == "# Full page content\nSome markdown text."
+        assert result["documents"][0].meta["url"] == "https://example.com/page"
+        assert result["documents"][0].meta["title"] == "Page Title"
+
+    def test_run_overrides_init_params_with_runtime_params(self, mock_client) -> None:
+        ws = FirecrawlWebSearch(
+            api_key=Secret.from_token("test-key"),
+            top_k=10,
+            search_params={"location": "US"},
+        )
+        ws._firecrawl_client = mock_client
+
+        ws.run(query="test", search_params={"location": "UK", "limit": 5})
+
+        mock_client.search.assert_called_once_with(
+            query="test",
+            location="UK",
+            limit=5,
+        )
+
+    @pytest.mark.asyncio
+    async def test_run_async(self, mock_async_client) -> None:
+        ws = FirecrawlWebSearch(api_key=Secret.from_token("test-key"), top_k=10)
+        ws._async_firecrawl_client = mock_async_client
+
+        result = await ws.run_async(query="test query")
+
+        assert len(result["documents"]) == 1
+        assert isinstance(result["documents"][0], Document)
+        assert result["links"] == ["https://example.com"]
+        mock_async_client.search.assert_awaited_once()
+
+    def test_run_returns_empty_on_error(self, mock_client) -> None:
+        mock_client.search.side_effect = Exception("API error")
+
+        ws = FirecrawlWebSearch(api_key=Secret.from_token("test-key"))
+        ws._firecrawl_client = mock_client
+
+        result = ws.run(query="test")
+        assert result["documents"] == []
+        assert result["links"] == []
+
+    @pytest.mark.asyncio
+    async def test_run_async_returns_empty_on_error(self, mock_async_client) -> None:
+        mock_async_client.search = AsyncMock(side_effect=Exception("API error"))
+
+        ws = FirecrawlWebSearch(api_key=Secret.from_token("test-key"))
+        ws._async_firecrawl_client = mock_async_client
+
+        result = await ws.run_async(query="test")
+        assert result["documents"] == []
+        assert result["links"] == []
+
+    def test_run_calls_warm_up(self, search_response) -> None:
+        with patch(
+            "haystack_integrations.components.websearch.firecrawl.firecrawl_websearch.Firecrawl"
+        ) as mock_firecrawl_client:
+            mock_firecrawl_client.return_value.search.return_value = search_response
+
+            ws = FirecrawlWebSearch(api_key=Secret.from_token("test-key"))
+            ws.run(query="test")
+
+            assert ws._firecrawl_client is mock_firecrawl_client.return_value
+            mock_firecrawl_client.assert_called_once_with(api_key="test-key")
+
+    @pytest.mark.asyncio
+    async def test_run_async_calls_warm_up(self, search_response) -> None:
+        with patch(
+            "haystack_integrations.components.websearch.firecrawl.firecrawl_websearch.AsyncFirecrawl"
+        ) as mock_async_firecrawl_client:
+            mock_async_firecrawl_client.return_value.search = AsyncMock(return_value=search_response)
+
+            ws = FirecrawlWebSearch(api_key=Secret.from_token("test-key"))
+            await ws.run_async(query="test")
+
+            assert ws._async_firecrawl_client is mock_async_firecrawl_client.return_value
+            mock_async_firecrawl_client.assert_called_once_with(api_key="test-key")
+
+    def test_run_empty_web_results(self, mock_client) -> None:
+        empty_response = MagicMock()
+        empty_response.web = []
+        mock_client.search.return_value = empty_response
+
+        ws = FirecrawlWebSearch(api_key=Secret.from_token("test-key"))
+        ws._firecrawl_client = mock_client
+        result = ws.run(query="obscure query")
+        assert result["documents"] == []
+        assert result["links"] == []
+
+    def test_run_none_web_results(self, mock_client) -> None:
+        none_response = MagicMock()
+        none_response.web = None
+        mock_client.search.return_value = none_response
+
+        ws = FirecrawlWebSearch(api_key=Secret.from_token("test-key"))
+        ws._firecrawl_client = mock_client
+        result = ws.run(query="obscure query")
+        assert result["documents"] == []
+        assert result["links"] == []
+
+
+@pytest.mark.skipif(
+    not os.environ.get("FIRECRAWL_API_KEY"),
+    reason="Export FIRECRAWL_API_KEY to run integration tests.",
+)
+@pytest.mark.integration
+class TestIntegration:
+    def test_run_integration(self) -> None:
+        ws = FirecrawlWebSearch(
+            api_key=Secret.from_env_var("FIRECRAWL_API_KEY"),
+            top_k=3,
+        )
+        result = ws.run(query="What is Haystack by deepset?")
+
+        assert "documents" in result
+        assert "links" in result
+        assert isinstance(result["documents"], list)
+        assert len(result["documents"]) > 0
+        assert len(result["links"]) > 0
+        assert isinstance(result["documents"][0], Document)
+        assert result["documents"][0].content
+
+    @pytest.mark.asyncio
+    async def test_run_async_integration(self) -> None:
+        ws = FirecrawlWebSearch(
+            api_key=Secret.from_env_var("FIRECRAWL_API_KEY"),
+            top_k=3,
+        )
+        result = await ws.run_async(query="What is Haystack by deepset?")
+
+        assert "documents" in result
+        assert "links" in result
+        assert len(result["documents"]) > 0
+        assert len(result["links"]) > 0

firecrawl_haystack-1.0.1/CHANGELOG.md

@@ -1,9 +0,0 @@
-# Changelog
-
-## [integrations/firecrawl-v1.0.0] - 2026-02-24
-
-### 🚀 Features
-
-- Add FirecrawlCrawler (#2859)
-
-<!-- generated by git-cliff -->