scrapedo-haystack 0.1.0__tar.gz

scrapedo_haystack-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,31 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install -e ".[test]"
+      - run: pytest -q
+
+  publish:
+    needs: test
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install build && python -m build
+      - uses: pypa/gh-action-pypi-publish@release/v1

scrapedo_haystack-0.1.0/.gitignore

@@ -0,0 +1,5 @@
+dist/
+__pycache__/
+*.egg-info/
+.pytest_cache/
+.venv/

scrapedo_haystack-0.1.0/LICENSE

@@ -0,0 +1,17 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   Copyright 2026 Scrape.do
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

scrapedo_haystack-0.1.0/PKG-INFO

@@ -0,0 +1,150 @@
+Metadata-Version: 2.5
+Name: scrapedo-haystack
+Version: 0.1.0
+Summary: Fetch web pages through the Scrape.do API in Haystack pipelines and agents.
+Project-URL: Homepage, https://scrape.do
+Project-URL: Documentation, https://scrape.do/documentation/
+Project-URL: Source, https://github.com/bugrahansss/scrapedo-haystack
+Project-URL: Issues, https://github.com/bugrahansss/scrapedo-haystack/issues
+Author: Scrape.do
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: haystack,llm,rag,scrape.do,web scraping
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Requires-Dist: haystack-ai>=2.0
+Requires-Dist: requests>=2.28
+Provides-Extra: test
+Requires-Dist: pytest; extra == 'test'
+Description-Content-Type: text/markdown
+
+# scrapedo-haystack
+
+[![PyPI](https://img.shields.io/pypi/v/scrapedo-haystack.svg)](https://pypi.org/project/scrapedo-haystack/)
+
+A [Haystack](https://haystack.deepset.ai/) component that fetches web pages through the [Scrape.do](https://scrape.do) API and returns them as Haystack `Document`s.
+
+Scrape.do handles proxies, retries, CAPTCHAs and anti-bot protection, so the component works on pages that block normal HTTP requests. Pages can be returned as Markdown (ready for an LLM) or raw HTML.
+
+## Installation
+
+```bash
+pip install scrapedo-haystack
+```
+
+Get an API token from [Scrape.do](https://scrape.do) and set it as an environment variable:
+
+```bash
+export SCRAPEDO_API_KEY="your-token"
+```
+
+## Usage
+
+### Fetch pages
+
+```python
+from scrapedo_haystack import ScrapeDoFetcher
+
+fetcher = ScrapeDoFetcher()
+result = fetcher.run(urls=["https://example.com"])
+
+doc = result["documents"][0]
+print(doc.meta["url"])
+print(doc.content)  # page content as Markdown
+```
+
+### Options
+
+```python
+fetcher = ScrapeDoFetcher(
+    output="markdown",   # "markdown" or "raw" (original HTML)
+    render=True,         # run JavaScript in a headless browser
+    super_proxy=True,    # use residential and mobile IPs
+    geo_code="us",       # route the request through a country
+    extra_params={"waitUntil": "networkidle0"},  # any other Scrape.do parameter
+    timeout=60,
+    max_workers=4,       # URLs fetched in parallel
+    raise_on_failure=False,  # skip failed URLs instead of raising
+)
+```
+
+All Scrape.do API parameters are listed in the [Scrape.do documentation](https://scrape.do/documentation/).
+
+### Use in a RAG pipeline
+
+Fetch pages, split them, and answer a question from their content:
+
+```python
+from haystack import Pipeline
+from haystack.components.builders import ChatPromptBuilder
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.dataclasses import ChatMessage
+
+from scrapedo_haystack import ScrapeDoFetcher
+
+template = [
+    ChatMessage.from_user(
+        """Answer the question using only these pages.
+
+{% for doc in documents %}
+Source: {{ doc.meta.url }}
+{{ doc.content }}
+{% endfor %}
+
+Question: {{ question }}"""
+    )
+]
+
+pipe = Pipeline()
+pipe.add_component("fetcher", ScrapeDoFetcher(render=True))
+pipe.add_component("splitter", DocumentSplitter(split_by="word", split_length=300))
+pipe.add_component("prompt", ChatPromptBuilder(template=template, required_variables=["question"]))
+pipe.add_component("llm", OpenAIChatGenerator(model="gpt-4o-mini"))
+
+pipe.connect("fetcher.documents", "splitter.documents")
+pipe.connect("splitter.documents", "prompt.documents")
+pipe.connect("prompt.prompt", "llm.messages")
+
+result = pipe.run({
+    "fetcher": {"urls": ["https://haystack.deepset.ai/overview/intro"]},
+    "prompt": {"question": "What is Haystack?"},
+})
+print(result["llm"]["replies"][0].text)
+```
+
+This example also needs `OPENAI_API_KEY` set.
+
+### Load a saved pipeline
+
+Haystack 3 only loads components from allowed modules when reading a pipeline from YAML. Add this package to the allowlist:
+
+```python
+pipe = Pipeline.loads(yaml_text, allowed_modules=["scrapedo_haystack.fetcher"])
+```
+
+The API token is never written to the YAML; it is read from `SCRAPEDO_API_KEY` when the pipeline loads.
+
+## Output
+
+Each fetched URL becomes one `Document`:
+
+| Field | Value |
+|---|---|
+| `content` | Page as Markdown, or raw HTML when `output="raw"` |
+| `meta["url"]` | The requested URL |
+| `meta["status_code"]` | HTTP status returned by Scrape.do |
+| `meta["content_type"]` | `text/markdown`, or the page's content type for raw output |
+
+## Development
+
+```bash
+pip install -e ".[test]"
+pytest
+```
+
+## License
+
+Apache-2.0

scrapedo_haystack-0.1.0/README.md

@@ -0,0 +1,128 @@
+# scrapedo-haystack
+
+[![PyPI](https://img.shields.io/pypi/v/scrapedo-haystack.svg)](https://pypi.org/project/scrapedo-haystack/)
+
+A [Haystack](https://haystack.deepset.ai/) component that fetches web pages through the [Scrape.do](https://scrape.do) API and returns them as Haystack `Document`s.
+
+Scrape.do handles proxies, retries, CAPTCHAs and anti-bot protection, so the component works on pages that block normal HTTP requests. Pages can be returned as Markdown (ready for an LLM) or raw HTML.
+
+## Installation
+
+```bash
+pip install scrapedo-haystack
+```
+
+Get an API token from [Scrape.do](https://scrape.do) and set it as an environment variable:
+
+```bash
+export SCRAPEDO_API_KEY="your-token"
+```
+
+## Usage
+
+### Fetch pages
+
+```python
+from scrapedo_haystack import ScrapeDoFetcher
+
+fetcher = ScrapeDoFetcher()
+result = fetcher.run(urls=["https://example.com"])
+
+doc = result["documents"][0]
+print(doc.meta["url"])
+print(doc.content)  # page content as Markdown
+```
+
+### Options
+
+```python
+fetcher = ScrapeDoFetcher(
+    output="markdown",   # "markdown" or "raw" (original HTML)
+    render=True,         # run JavaScript in a headless browser
+    super_proxy=True,    # use residential and mobile IPs
+    geo_code="us",       # route the request through a country
+    extra_params={"waitUntil": "networkidle0"},  # any other Scrape.do parameter
+    timeout=60,
+    max_workers=4,       # URLs fetched in parallel
+    raise_on_failure=False,  # skip failed URLs instead of raising
+)
+```
+
+All Scrape.do API parameters are listed in the [Scrape.do documentation](https://scrape.do/documentation/).
+
+### Use in a RAG pipeline
+
+Fetch pages, split them, and answer a question from their content:
+
+```python
+from haystack import Pipeline
+from haystack.components.builders import ChatPromptBuilder
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.dataclasses import ChatMessage
+
+from scrapedo_haystack import ScrapeDoFetcher
+
+template = [
+    ChatMessage.from_user(
+        """Answer the question using only these pages.
+
+{% for doc in documents %}
+Source: {{ doc.meta.url }}
+{{ doc.content }}
+{% endfor %}
+
+Question: {{ question }}"""
+    )
+]
+
+pipe = Pipeline()
+pipe.add_component("fetcher", ScrapeDoFetcher(render=True))
+pipe.add_component("splitter", DocumentSplitter(split_by="word", split_length=300))
+pipe.add_component("prompt", ChatPromptBuilder(template=template, required_variables=["question"]))
+pipe.add_component("llm", OpenAIChatGenerator(model="gpt-4o-mini"))
+
+pipe.connect("fetcher.documents", "splitter.documents")
+pipe.connect("splitter.documents", "prompt.documents")
+pipe.connect("prompt.prompt", "llm.messages")
+
+result = pipe.run({
+    "fetcher": {"urls": ["https://haystack.deepset.ai/overview/intro"]},
+    "prompt": {"question": "What is Haystack?"},
+})
+print(result["llm"]["replies"][0].text)
+```
+
+This example also needs `OPENAI_API_KEY` set.
+
+### Load a saved pipeline
+
+Haystack 3 only loads components from allowed modules when reading a pipeline from YAML. Add this package to the allowlist:
+
+```python
+pipe = Pipeline.loads(yaml_text, allowed_modules=["scrapedo_haystack.fetcher"])
+```
+
+The API token is never written to the YAML; it is read from `SCRAPEDO_API_KEY` when the pipeline loads.
+
+## Output
+
+Each fetched URL becomes one `Document`:
+
+| Field | Value |
+|---|---|
+| `content` | Page as Markdown, or raw HTML when `output="raw"` |
+| `meta["url"]` | The requested URL |
+| `meta["status_code"]` | HTTP status returned by Scrape.do |
+| `meta["content_type"]` | `text/markdown`, or the page's content type for raw output |
+
+## Development
+
+```bash
+pip install -e ".[test]"
+pytest
+```
+
+## License
+
+Apache-2.0

scrapedo_haystack-0.1.0/pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "scrapedo-haystack"
+version = "0.1.0"
+description = "Fetch web pages through the Scrape.do API in Haystack pipelines and agents."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+authors = [{ name = "Scrape.do" }]
+keywords = ["haystack", "scrape.do", "web scraping", "rag", "llm"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: OS Independent",
+]
+dependencies = ["haystack-ai>=2.0", "requests>=2.28"]
+
+[project.optional-dependencies]
+test = ["pytest"]
+
+[project.urls]
+Homepage = "https://scrape.do"
+Documentation = "https://scrape.do/documentation/"
+Source = "https://github.com/bugrahansss/scrapedo-haystack"
+Issues = "https://github.com/bugrahansss/scrapedo-haystack/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/scrapedo_haystack"]

scrapedo_haystack-0.1.0/src/scrapedo_haystack/__init__.py

@@ -0,0 +1,4 @@
+from scrapedo_haystack.fetcher import ScrapeDoFetcher
+
+__all__ = ["ScrapeDoFetcher"]
+__version__ = "0.1.0"

scrapedo_haystack-0.1.0/src/scrapedo_haystack/fetcher.py

@@ -0,0 +1,144 @@
+"""Haystack component that fetches web pages through the Scrape.do API."""
+
+from __future__ import annotations
+
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Literal
+
+import requests
+from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.utils import Secret, deserialize_secrets_inplace
+
+logger = logging.getLogger(__name__)
+
+SCRAPEDO_API_URL = "https://api.scrape.do/"
+
+OutputFormat = Literal["markdown", "raw"]
+
+
+@component
+class ScrapeDoFetcher:
+    """
+    Fetches web pages through the Scrape.do API and returns them as Haystack `Document`s.
+
+    Scrape.do handles proxies, retries and anti-bot protection. Set `render=True` for pages
+    that need JavaScript, `super_proxy=True` for residential and mobile IPs, and `geo_code`
+    to route the request through a specific country.
+
+    Usage example:
+    ```python
+    from scrapedo_haystack import ScrapeDoFetcher
+
+    fetcher = ScrapeDoFetcher()  # reads the SCRAPEDO_API_KEY environment variable
+    result = fetcher.run(urls=["https://example.com"])
+    print(result["documents"][0].content)
+    ```
+    """
+
+    def __init__(
+        self,
+        api_key: Secret = Secret.from_env_var("SCRAPEDO_API_KEY"),
+        output: OutputFormat = "markdown",
+        render: bool = False,
+        super_proxy: bool = False,
+        geo_code: str | None = None,
+        extra_params: dict[str, Any] | None = None,
+        timeout: int = 60,
+        max_workers: int = 4,
+        raise_on_failure: bool = False,
+    ) -> None:
+        """
+        Create a ScrapeDoFetcher.
+
+        :param api_key: Scrape.do API token. Defaults to the `SCRAPEDO_API_KEY` environment variable.
+        :param output: `"markdown"` returns the page as Markdown, `"raw"` returns the original HTML.
+        :param render: Render the page in a headless browser so JavaScript runs.
+        :param super_proxy: Use residential and mobile proxies.
+        :param geo_code: Two-letter country code to route the request through, for example `"us"`.
+        :param extra_params: Any other Scrape.do API parameters, passed as-is (for example `{"waitUntil": "networkidle0"}`).
+        :param timeout: Request timeout in seconds.
+        :param max_workers: Number of URLs fetched in parallel.
+        :param raise_on_failure: Raise an error when a URL fails. If `False`, failed URLs are logged and skipped.
+        """
+        if output not in ("markdown", "raw"):
+            raise ValueError(f"output must be 'markdown' or 'raw', got {output!r}")
+        self.api_key = api_key
+        self.output = output
+        self.render = render
+        self.super_proxy = super_proxy
+        self.geo_code = geo_code
+        self.extra_params = extra_params or {}
+        self.timeout = timeout
+        self.max_workers = max_workers
+        self.raise_on_failure = raise_on_failure
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the component to a dictionary."""
+        return default_to_dict(
+            self,
+            api_key=self.api_key.to_dict(),
+            output=self.output,
+            render=self.render,
+            super_proxy=self.super_proxy,
+            geo_code=self.geo_code,
+            extra_params=self.extra_params,
+            timeout=self.timeout,
+            max_workers=self.max_workers,
+            raise_on_failure=self.raise_on_failure,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ScrapeDoFetcher":
+        """Deserialize the component from a dictionary."""
+        deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
+        return default_from_dict(cls, data)
+
+    def _build_params(self, url: str) -> dict[str, Any]:
+        token = self.api_key.resolve_value()
+        if not token:
+            raise ValueError("Scrape.do API key is missing. Set the SCRAPEDO_API_KEY environment variable.")
+        params: dict[str, Any] = {"token": token, "url": url}
+        if self.output == "markdown":
+            params["output"] = "markdown"
+        if self.render:
+            params["render"] = "true"
+        if self.super_proxy:
+            params["super"] = "true"
+        if self.geo_code:
+            params["geoCode"] = self.geo_code
+        params.update(self.extra_params)
+        return params
+
+    def _fetch(self, url: str) -> Document | None:
+        try:
+            response = requests.get(SCRAPEDO_API_URL, params=self._build_params(url), timeout=self.timeout)
+            response.raise_for_status()
+        except Exception as error:
+            if self.raise_on_failure:
+                raise
+            # never log the request URL: it contains the API token
+            logger.warning("Scrape.do could not fetch %s: %s", url, type(error).__name__)
+            return None
+
+        meta = {
+            "url": url,
+            "status_code": response.status_code,
+            "content_type": "text/markdown" if self.output == "markdown" else response.headers.get("Content-Type"),
+        }
+        return Document(content=response.text, meta=meta)
+
+    @component.output_types(documents=list[Document])
+    def run(self, urls: list[str]) -> dict[str, list[Document]]:
+        """
+        Fetch the given URLs through Scrape.do.
+
+        :param urls: The web pages to fetch.
+        :returns: A dictionary with `documents`: one Document per successfully fetched URL,
+            with the page content and `url`, `status_code` and `content_type` in `meta`.
+        """
+        if not urls:
+            return {"documents": []}
+        with ThreadPoolExecutor(max_workers=min(self.max_workers, len(urls))) as pool:
+            results = list(pool.map(self._fetch, urls))
+        return {"documents": [doc for doc in results if doc is not None]}

scrapedo_haystack-0.1.0/tests/test_fetcher.py

@@ -0,0 +1,132 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+import requests
+from haystack import Document, Pipeline
+from haystack.utils import Secret
+
+from scrapedo_haystack import ScrapeDoFetcher
+
+
+def _response(text="# Hello", status=200, content_type="text/plain"):
+    resp = MagicMock()
+    resp.text = text
+    resp.status_code = status
+    resp.headers = {"Content-Type": content_type}
+    if status >= 400:
+        resp.raise_for_status.side_effect = requests.HTTPError(f"{status}")
+    return resp
+
+
+@pytest.fixture
+def fetcher():
+    return ScrapeDoFetcher(api_key=Secret.from_token("test-token"))
+
+
+@patch("scrapedo_haystack.fetcher.requests.get")
+def test_returns_documents(mock_get, fetcher):
+    mock_get.return_value = _response("# Example page")
+    docs = fetcher.run(urls=["https://example.com"])["documents"]
+
+    assert len(docs) == 1
+    assert isinstance(docs[0], Document)
+    assert docs[0].content == "# Example page"
+    assert docs[0].meta["url"] == "https://example.com"
+    assert docs[0].meta["content_type"] == "text/markdown"
+
+
+@patch("scrapedo_haystack.fetcher.requests.get")
+def test_sends_scrapedo_params(mock_get):
+    mock_get.return_value = _response()
+    fetcher = ScrapeDoFetcher(
+        api_key=Secret.from_token("test-token"),
+        render=True,
+        super_proxy=True,
+        geo_code="us",
+        extra_params={"waitUntil": "networkidle0"},
+    )
+    fetcher.run(urls=["https://example.com"])
+
+    args, kwargs = mock_get.call_args
+    assert args[0] == "https://api.scrape.do/"
+    assert kwargs["params"] == {
+        "token": "test-token",
+        "url": "https://example.com",
+        "output": "markdown",
+        "render": "true",
+        "super": "true",
+        "geoCode": "us",
+        "waitUntil": "networkidle0",
+    }
+
+
+@patch("scrapedo_haystack.fetcher.requests.get")
+def test_raw_output_has_no_output_param(mock_get):
+    mock_get.return_value = _response("<html></html>", content_type="text/html")
+    fetcher = ScrapeDoFetcher(api_key=Secret.from_token("t"), output="raw")
+    docs = fetcher.run(urls=["https://example.com"])["documents"]
+
+    assert "output" not in mock_get.call_args.kwargs["params"]
+    assert docs[0].meta["content_type"] == "text/html"
+
+
+@patch("scrapedo_haystack.fetcher.requests.get")
+def test_failed_url_is_skipped(mock_get, fetcher):
+    mock_get.side_effect = [_response("ok"), _response(status=500)]
+    docs = fetcher.run(urls=["https://a.com", "https://b.com"])["documents"]
+    assert [d.meta["url"] for d in docs] == ["https://a.com"]
+
+
+@patch("scrapedo_haystack.fetcher.requests.get")
+def test_raise_on_failure(mock_get):
+    mock_get.return_value = _response(status=401)
+    fetcher = ScrapeDoFetcher(api_key=Secret.from_token("t"), raise_on_failure=True)
+    with pytest.raises(requests.HTTPError):
+        fetcher.run(urls=["https://example.com"])
+
+
+def test_empty_urls(fetcher):
+    assert fetcher.run(urls=[]) == {"documents": []}
+
+
+def test_missing_api_key(monkeypatch):
+    monkeypatch.delenv("SCRAPEDO_API_KEY", raising=False)
+    fetcher = ScrapeDoFetcher(api_key=Secret.from_env_var("SCRAPEDO_API_KEY", strict=False), raise_on_failure=True)
+    with pytest.raises(ValueError, match="API key is missing"):
+        fetcher.run(urls=["https://example.com"])
+
+
+def test_invalid_output():
+    with pytest.raises(ValueError):
+        ScrapeDoFetcher(api_key=Secret.from_token("t"), output="pdf")
+
+
+def test_serialization_roundtrip(monkeypatch):
+    monkeypatch.setenv("SCRAPEDO_API_KEY", "env-token")
+    fetcher = ScrapeDoFetcher(render=True, geo_code="de")
+    data = fetcher.to_dict()
+
+    assert "env-token" not in str(data)
+    restored = ScrapeDoFetcher.from_dict(data)
+    assert restored.render is True
+    assert restored.geo_code == "de"
+    assert restored.api_key.resolve_value() == "env-token"
+
+
+@patch("scrapedo_haystack.fetcher.requests.get")
+def test_works_in_pipeline_and_yaml(mock_get, monkeypatch):
+    monkeypatch.setenv("SCRAPEDO_API_KEY", "env-token")
+    mock_get.return_value = _response("# Page")
+
+    from haystack.components.preprocessors import DocumentSplitter
+
+    pipe = Pipeline()
+    pipe.add_component("fetcher", ScrapeDoFetcher())
+    pipe.add_component("splitter", DocumentSplitter(split_by="word", split_length=50))
+    pipe.connect("fetcher.documents", "splitter.documents")
+
+    result = pipe.run({"fetcher": {"urls": ["https://example.com"]}})
+    assert result["splitter"]["documents"][0].content == "# Page"
+
+    restored = Pipeline.loads(pipe.dumps(), allowed_modules=["scrapedo_haystack.fetcher"])
+    assert "fetcher" in restored.graph.nodes