scrapedo-haystack 0.1.0__py3-none-any.whl

scrapedo_haystack/__init__.py

@@ -0,0 +1,4 @@
+from scrapedo_haystack.fetcher import ScrapeDoFetcher
+
+__all__ = ["ScrapeDoFetcher"]
+__version__ = "0.1.0"

scrapedo_haystack/fetcher.py

@@ -0,0 +1,144 @@
+"""Haystack component that fetches web pages through the Scrape.do API."""
+
+from __future__ import annotations
+
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Literal
+
+import requests
+from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.utils import Secret, deserialize_secrets_inplace
+
+logger = logging.getLogger(__name__)
+
+SCRAPEDO_API_URL = "https://api.scrape.do/"
+
+OutputFormat = Literal["markdown", "raw"]
+
+
+@component
+class ScrapeDoFetcher:
+    """
+    Fetches web pages through the Scrape.do API and returns them as Haystack `Document`s.
+
+    Scrape.do handles proxies, retries and anti-bot protection. Set `render=True` for pages
+    that need JavaScript, `super_proxy=True` for residential and mobile IPs, and `geo_code`
+    to route the request through a specific country.
+
+    Usage example:
+    ```python
+    from scrapedo_haystack import ScrapeDoFetcher
+
+    fetcher = ScrapeDoFetcher()  # reads the SCRAPEDO_API_KEY environment variable
+    result = fetcher.run(urls=["https://example.com"])
+    print(result["documents"][0].content)
+    ```
+    """
+
+    def __init__(
+        self,
+        api_key: Secret = Secret.from_env_var("SCRAPEDO_API_KEY"),
+        output: OutputFormat = "markdown",
+        render: bool = False,
+        super_proxy: bool = False,
+        geo_code: str | None = None,
+        extra_params: dict[str, Any] | None = None,
+        timeout: int = 60,
+        max_workers: int = 4,
+        raise_on_failure: bool = False,
+    ) -> None:
+        """
+        Create a ScrapeDoFetcher.
+
+        :param api_key: Scrape.do API token. Defaults to the `SCRAPEDO_API_KEY` environment variable.
+        :param output: `"markdown"` returns the page as Markdown, `"raw"` returns the original HTML.
+        :param render: Render the page in a headless browser so JavaScript runs.
+        :param super_proxy: Use residential and mobile proxies.
+        :param geo_code: Two-letter country code to route the request through, for example `"us"`.
+        :param extra_params: Any other Scrape.do API parameters, passed as-is (for example `{"waitUntil": "networkidle0"}`).
+        :param timeout: Request timeout in seconds.
+        :param max_workers: Number of URLs fetched in parallel.
+        :param raise_on_failure: Raise an error when a URL fails. If `False`, failed URLs are logged and skipped.
+        """
+        if output not in ("markdown", "raw"):
+            raise ValueError(f"output must be 'markdown' or 'raw', got {output!r}")
+        self.api_key = api_key
+        self.output = output
+        self.render = render
+        self.super_proxy = super_proxy
+        self.geo_code = geo_code
+        self.extra_params = extra_params or {}
+        self.timeout = timeout
+        self.max_workers = max_workers
+        self.raise_on_failure = raise_on_failure
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the component to a dictionary."""
+        return default_to_dict(
+            self,
+            api_key=self.api_key.to_dict(),
+            output=self.output,
+            render=self.render,
+            super_proxy=self.super_proxy,
+            geo_code=self.geo_code,
+            extra_params=self.extra_params,
+            timeout=self.timeout,
+            max_workers=self.max_workers,
+            raise_on_failure=self.raise_on_failure,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ScrapeDoFetcher":
+        """Deserialize the component from a dictionary."""
+        deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
+        return default_from_dict(cls, data)
+
+    def _build_params(self, url: str) -> dict[str, Any]:
+        token = self.api_key.resolve_value()
+        if not token:
+            raise ValueError("Scrape.do API key is missing. Set the SCRAPEDO_API_KEY environment variable.")
+        params: dict[str, Any] = {"token": token, "url": url}
+        if self.output == "markdown":
+            params["output"] = "markdown"
+        if self.render:
+            params["render"] = "true"
+        if self.super_proxy:
+            params["super"] = "true"
+        if self.geo_code:
+            params["geoCode"] = self.geo_code
+        params.update(self.extra_params)
+        return params
+
+    def _fetch(self, url: str) -> Document | None:
+        try:
+            response = requests.get(SCRAPEDO_API_URL, params=self._build_params(url), timeout=self.timeout)
+            response.raise_for_status()
+        except Exception as error:
+            if self.raise_on_failure:
+                raise
+            # never log the request URL: it contains the API token
+            logger.warning("Scrape.do could not fetch %s: %s", url, type(error).__name__)
+            return None
+
+        meta = {
+            "url": url,
+            "status_code": response.status_code,
+            "content_type": "text/markdown" if self.output == "markdown" else response.headers.get("Content-Type"),
+        }
+        return Document(content=response.text, meta=meta)
+
+    @component.output_types(documents=list[Document])
+    def run(self, urls: list[str]) -> dict[str, list[Document]]:
+        """
+        Fetch the given URLs through Scrape.do.
+
+        :param urls: The web pages to fetch.
+        :returns: A dictionary with `documents`: one Document per successfully fetched URL,
+            with the page content and `url`, `status_code` and `content_type` in `meta`.
+        """
+        if not urls:
+            return {"documents": []}
+        with ThreadPoolExecutor(max_workers=min(self.max_workers, len(urls))) as pool:
+            results = list(pool.map(self._fetch, urls))
+        return {"documents": [doc for doc in results if doc is not None]}

scrapedo_haystack-0.1.0.dist-info/METADATA

@@ -0,0 +1,150 @@
+Metadata-Version: 2.5
+Name: scrapedo-haystack
+Version: 0.1.0
+Summary: Fetch web pages through the Scrape.do API in Haystack pipelines and agents.
+Project-URL: Homepage, https://scrape.do
+Project-URL: Documentation, https://scrape.do/documentation/
+Project-URL: Source, https://github.com/bugrahansss/scrapedo-haystack
+Project-URL: Issues, https://github.com/bugrahansss/scrapedo-haystack/issues
+Author: Scrape.do
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: haystack,llm,rag,scrape.do,web scraping
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Requires-Dist: haystack-ai>=2.0
+Requires-Dist: requests>=2.28
+Provides-Extra: test
+Requires-Dist: pytest; extra == 'test'
+Description-Content-Type: text/markdown
+
+# scrapedo-haystack
+
+[![PyPI](https://img.shields.io/pypi/v/scrapedo-haystack.svg)](https://pypi.org/project/scrapedo-haystack/)
+
+A [Haystack](https://haystack.deepset.ai/) component that fetches web pages through the [Scrape.do](https://scrape.do) API and returns them as Haystack `Document`s.
+
+Scrape.do handles proxies, retries, CAPTCHAs and anti-bot protection, so the component works on pages that block normal HTTP requests. Pages can be returned as Markdown (ready for an LLM) or raw HTML.
+
+## Installation
+
+```bash
+pip install scrapedo-haystack
+```
+
+Get an API token from [Scrape.do](https://scrape.do) and set it as an environment variable:
+
+```bash
+export SCRAPEDO_API_KEY="your-token"
+```
+
+## Usage
+
+### Fetch pages
+
+```python
+from scrapedo_haystack import ScrapeDoFetcher
+
+fetcher = ScrapeDoFetcher()
+result = fetcher.run(urls=["https://example.com"])
+
+doc = result["documents"][0]
+print(doc.meta["url"])
+print(doc.content)  # page content as Markdown
+```
+
+### Options
+
+```python
+fetcher = ScrapeDoFetcher(
+    output="markdown",   # "markdown" or "raw" (original HTML)
+    render=True,         # run JavaScript in a headless browser
+    super_proxy=True,    # use residential and mobile IPs
+    geo_code="us",       # route the request through a country
+    extra_params={"waitUntil": "networkidle0"},  # any other Scrape.do parameter
+    timeout=60,
+    max_workers=4,       # URLs fetched in parallel
+    raise_on_failure=False,  # skip failed URLs instead of raising
+)
+```
+
+All Scrape.do API parameters are listed in the [Scrape.do documentation](https://scrape.do/documentation/).
+
+### Use in a RAG pipeline
+
+Fetch pages, split them, and answer a question from their content:
+
+```python
+from haystack import Pipeline
+from haystack.components.builders import ChatPromptBuilder
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.dataclasses import ChatMessage
+
+from scrapedo_haystack import ScrapeDoFetcher
+
+template = [
+    ChatMessage.from_user(
+        """Answer the question using only these pages.
+
+{% for doc in documents %}
+Source: {{ doc.meta.url }}
+{{ doc.content }}
+{% endfor %}
+
+Question: {{ question }}"""
+    )
+]
+
+pipe = Pipeline()
+pipe.add_component("fetcher", ScrapeDoFetcher(render=True))
+pipe.add_component("splitter", DocumentSplitter(split_by="word", split_length=300))
+pipe.add_component("prompt", ChatPromptBuilder(template=template, required_variables=["question"]))
+pipe.add_component("llm", OpenAIChatGenerator(model="gpt-4o-mini"))
+
+pipe.connect("fetcher.documents", "splitter.documents")
+pipe.connect("splitter.documents", "prompt.documents")
+pipe.connect("prompt.prompt", "llm.messages")
+
+result = pipe.run({
+    "fetcher": {"urls": ["https://haystack.deepset.ai/overview/intro"]},
+    "prompt": {"question": "What is Haystack?"},
+})
+print(result["llm"]["replies"][0].text)
+```
+
+This example also needs `OPENAI_API_KEY` set.
+
+### Load a saved pipeline
+
+Haystack 3 only loads components from allowed modules when reading a pipeline from YAML. Add this package to the allowlist:
+
+```python
+pipe = Pipeline.loads(yaml_text, allowed_modules=["scrapedo_haystack.fetcher"])
+```
+
+The API token is never written to the YAML; it is read from `SCRAPEDO_API_KEY` when the pipeline loads.
+
+## Output
+
+Each fetched URL becomes one `Document`:
+
+| Field | Value |
+|---|---|
+| `content` | Page as Markdown, or raw HTML when `output="raw"` |
+| `meta["url"]` | The requested URL |
+| `meta["status_code"]` | HTTP status returned by Scrape.do |
+| `meta["content_type"]` | `text/markdown`, or the page's content type for raw output |
+
+## Development
+
+```bash
+pip install -e ".[test]"
+pytest
+```
+
+## License
+
+Apache-2.0

scrapedo_haystack-0.1.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+scrapedo_haystack/__init__.py,sha256=i5LrKHQBM5d6fuJfio0DXCv2icDL63IyiamxLI2EXEU,107
+scrapedo_haystack/fetcher.py,sha256=dYx540GIkch1OaNlxXE5Ie07-wJn2ixEp70oThh-_m4,5751
+scrapedo_haystack-0.1.0.dist-info/METADATA,sha256=f_pzB1hV8VOVDr-5zgO0x8sAYcdzlPs3uPPU-j0k4S0,4595
+scrapedo_haystack-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+scrapedo_haystack-0.1.0.dist-info/licenses/LICENSE,sha256=S8lp3ijo6jkSfxEyZIQIh6Wnu1t7IGcfoLZd59rNPTs,738
+scrapedo_haystack-0.1.0.dist-info/RECORD,,

scrapedo_haystack-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

scrapedo_haystack-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,17 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   Copyright 2026 Scrape.do
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.