langchain-hydrafetch 0.1.0__py3-none-any.whl

langchain_hydrafetch/__init__.py

@@ -0,0 +1,20 @@
+"""LangChain integration for the Hydrafetch web data API."""
+
+from .document_loaders import HydrafetchLoader
+from .retrievers import HydrafetchSearchRetriever
+from .tools import (
+    HydrafetchBrandTool,
+    HydrafetchExtractTool,
+    HydrafetchScrapeTool,
+    HydrafetchSearchTool,
+)
+
+__all__ = [
+    "HydrafetchBrandTool",
+    "HydrafetchExtractTool",
+    "HydrafetchLoader",
+    "HydrafetchScrapeTool",
+    "HydrafetchSearchRetriever",
+    "HydrafetchSearchTool",
+]
+__version__ = "0.1.0"

langchain_hydrafetch/_common.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from hydrafetch import AsyncHydrafetch, Hydrafetch
+from langchain_core.documents import Document
+
+CONTENT_FORMATS = ("markdown", "html", "rawHtml", "summary")
+
+_METADATA_KEYS = (
+    "title",
+    "description",
+    "language",
+    "siteName",
+    "author",
+    "publishedTime",
+    "wordCount",
+    "pageType",
+    "image",
+)
+
+
+def build_client(
+    api_key: str | None,
+    base_url: str | None,
+    timeout: float | None,
+    max_retries: int | None,
+) -> Hydrafetch:
+    return Hydrafetch(api_key, **_client_kwargs(base_url, timeout, max_retries))
+
+
+def build_async_client(
+    api_key: str | None,
+    base_url: str | None,
+    timeout: float | None,
+    max_retries: int | None,
+) -> AsyncHydrafetch:
+    return AsyncHydrafetch(api_key, **_client_kwargs(base_url, timeout, max_retries))
+
+
+def _client_kwargs(
+    base_url: str | None, timeout: float | None, max_retries: int | None
+) -> dict[str, Any]:
+    kwargs: dict[str, Any] = {}
+    if base_url is not None:
+        kwargs["base_url"] = base_url
+    if timeout is not None:
+        kwargs["timeout"] = timeout
+    if max_retries is not None:
+        kwargs["max_retries"] = max_retries
+    return kwargs
+
+
+def unwrap(payload: Any) -> dict[str, Any]:
+    if isinstance(payload, dict) and isinstance(payload.get("data"), dict):
+        return payload["data"]
+    return payload if isinstance(payload, dict) else {}
+
+
+def page_to_document(data: Mapping[str, Any], content_format: str) -> Document:
+    content = data.get(content_format)
+    if content is None and content_format != "markdown":
+        content = data.get("markdown")
+    if content is None:
+        structured = data.get("structured") or data.get("json")
+        content = str(structured) if structured is not None else ""
+
+    metadata: dict[str, Any] = {"source": data.get("url") or data.get("finalUrl") or ""}
+    if data.get("finalUrl") and data.get("finalUrl") != metadata["source"]:
+        metadata["final_url"] = data["finalUrl"]
+    if data.get("status") is not None:
+        metadata["status"] = data["status"]
+
+    raw_metadata = data.get("metadata")
+    if isinstance(raw_metadata, Mapping):
+        for key in _METADATA_KEYS:
+            value = raw_metadata.get(key)
+            if value is not None:
+                metadata[_snake(key)] = value
+
+    return Document(page_content=str(content), metadata=metadata)
+
+
+def _snake(key: str) -> str:
+    out: list[str] = []
+    for char in key:
+        if char.isupper():
+            out.append("_")
+            out.append(char.lower())
+        else:
+            out.append(char)
+    return "".join(out)

langchain_hydrafetch/document_loaders.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any, Literal
+
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+
+from ._common import build_client, page_to_document, unwrap
+
+Mode = Literal["scrape", "crawl", "map"]
+
+
+class HydrafetchLoader(BaseLoader):
+    """Load web pages as LangChain documents via the Hydrafetch API."""
+
+    def __init__(
+        self,
+        url: str,
+        *,
+        mode: Mode = "scrape",
+        api_key: str | None = None,
+        base_url: str | None = None,
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        content_format: str = "markdown",
+        raise_for_status: bool = True,
+        params: dict[str, Any] | None = None,
+    ) -> None:
+        if mode not in ("scrape", "crawl", "map"):
+            raise ValueError(f"mode must be 'scrape', 'crawl' or 'map', got {mode!r}")
+        self.url = url
+        self.mode = mode
+        self.content_format = content_format
+        self.raise_for_status = raise_for_status
+        self.params = params or {}
+        self._client = build_client(api_key, base_url, timeout, max_retries)
+
+    def lazy_load(self) -> Iterator[Document]:
+        if self.mode == "scrape":
+            yield from self._load_scrape()
+        elif self.mode == "map":
+            yield from self._load_map()
+        else:
+            yield from self._load_crawl()
+
+    def _load_scrape(self) -> Iterator[Document]:
+        data = unwrap(self._client.scrape(self.url, **self._scrape_params()))
+        if self.raise_for_status and not _ok(data):
+            raise ValueError(
+                f"{self.url} returned HTTP {data.get('status')}. The body of an error page is "
+                "not the page you asked for; pass raise_for_status=False to load it anyway."
+            )
+        yield page_to_document(data, self.content_format)
+
+    def _load_map(self) -> Iterator[Document]:
+        data = unwrap(self._client.map(self.url, **self.params))
+        for entry in data.get("links") or []:
+            if not isinstance(entry, dict):
+                continue
+            found = entry.get("url")
+            if not found:
+                continue
+            metadata: dict[str, Any] = {"source": found}
+            if entry.get("lastmod"):
+                metadata["lastmod"] = entry["lastmod"]
+            yield Document(page_content=found, metadata=metadata)
+
+    def _load_crawl(self) -> Iterator[Document]:
+        job = unwrap(self._client.crawl(self.url, **self._scrape_params()))
+        for page in job.get("pages") or []:
+            if not isinstance(page, dict):
+                continue
+            data = page.get("data")
+            if not isinstance(data, dict):
+                continue
+            data.setdefault("url", page.get("url") or page.get("requestedUrl"))
+            if self.raise_for_status and not _ok(data):
+                continue
+            document = page_to_document(data, self.content_format)
+            if page.get("depth") is not None:
+                document.metadata["depth"] = page["depth"]
+            yield document
+
+    def _scrape_params(self) -> dict[str, Any]:
+        params = dict(self.params)
+        params.setdefault("formats", [self.content_format])
+        return params
+
+
+def _ok(data: dict[str, Any]) -> bool:
+    status = data.get("status")
+    if status is None:
+        return True
+    try:
+        return 200 <= int(status) < 300
+    except (TypeError, ValueError):
+        return True

langchain_hydrafetch/retrievers.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+from typing import Any
+
+from langchain_core.callbacks import (
+    AsyncCallbackManagerForRetrieverRun,
+    CallbackManagerForRetrieverRun,
+)
+from langchain_core.documents import Document
+from langchain_core.retrievers import BaseRetriever
+from pydantic import ConfigDict, Field, PrivateAttr
+
+from ._common import build_async_client, build_client, unwrap
+
+
+class HydrafetchSearchRetriever(BaseRetriever):
+    """Retrieve web results for a query using Hydrafetch's first-party search."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    k: int = 5
+    scrape_content: bool = False
+    content_format: str = "markdown"
+    search_params: dict[str, Any] = Field(default_factory=dict)
+
+    api_key: str | None = None
+    base_url: str | None = None
+    timeout: float | None = None
+    max_retries: int | None = None
+
+    _client: Any = PrivateAttr(default=None)
+    _aclient: Any = PrivateAttr(default=None)
+
+    def _sync(self) -> Any:
+        if self._client is None:
+            self._client = build_client(self.api_key, self.base_url, self.timeout, self.max_retries)
+        return self._client
+
+    def _async(self) -> Any:
+        if self._aclient is None:
+            self._aclient = build_async_client(
+                self.api_key, self.base_url, self.timeout, self.max_retries
+            )
+        return self._aclient
+
+    def _params(self, k: int | None) -> dict[str, Any]:
+        params = dict(self.search_params)
+        params.setdefault("limit", k or self.k)
+        return params
+
+    def _get_relevant_documents(
+        self,
+        query: str,
+        *,
+        run_manager: CallbackManagerForRetrieverRun,
+        k: int | None = None,
+        **kwargs: Any,
+    ) -> list[Document]:
+        data = unwrap(self._sync().search(query, **self._params(k)))
+        documents = [_result_to_document(item) for item in _results(data)]
+        if not self.scrape_content:
+            return documents[: k or self.k]
+        return [self._with_content(doc) for doc in documents[: k or self.k]]
+
+    async def _aget_relevant_documents(
+        self,
+        query: str,
+        *,
+        run_manager: AsyncCallbackManagerForRetrieverRun,
+        k: int | None = None,
+        **kwargs: Any,
+    ) -> list[Document]:
+        data = unwrap(await self._async().search(query, **self._params(k)))
+        documents = [_result_to_document(item) for item in _results(data)]
+        if not self.scrape_content:
+            return documents[: k or self.k]
+        return [await self._awith_content(doc) for doc in documents[: k or self.k]]
+
+    def _with_content(self, document: Document) -> Document:
+        source = document.metadata.get("source")
+        if not source:
+            return document
+        page = unwrap(self._sync().scrape(source, formats=[self.content_format]))
+        content = page.get(self.content_format)
+        if content:
+            document.page_content = str(content)
+        return document
+
+    async def _awith_content(self, document: Document) -> Document:
+        source = document.metadata.get("source")
+        if not source:
+            return document
+        page = unwrap(await self._async().scrape(source, formats=[self.content_format]))
+        content = page.get(self.content_format)
+        if content:
+            document.page_content = str(content)
+        return document
+
+
+def _results(data: dict[str, Any]) -> list[dict[str, Any]]:
+    return [item for item in (data.get("results") or []) if isinstance(item, dict)]
+
+
+def _result_to_document(item: dict[str, Any]) -> Document:
+    metadata: dict[str, Any] = {"source": item.get("url") or ""}
+    if item.get("title"):
+        metadata["title"] = item["title"]
+    if item.get("rank") is not None:
+        metadata["rank"] = item["rank"]
+    return Document(page_content=str(item.get("snippet") or ""), metadata=metadata)

langchain_hydrafetch/tools.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from langchain_core.callbacks import (
+    AsyncCallbackManagerForToolRun,
+    CallbackManagerForToolRun,
+)
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
+
+from ._common import build_async_client, build_client, unwrap
+
+
+class _HydrafetchTool(BaseTool):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    api_key: str | None = None
+    base_url: str | None = None
+    timeout: float | None = None
+    max_retries: int | None = None
+
+    _client: Any = PrivateAttr(default=None)
+    _aclient: Any = PrivateAttr(default=None)
+
+    def _sync(self) -> Any:
+        if self._client is None:
+            self._client = build_client(self.api_key, self.base_url, self.timeout, self.max_retries)
+        return self._client
+
+    def _async(self) -> Any:
+        if self._aclient is None:
+            self._aclient = build_async_client(
+                self.api_key, self.base_url, self.timeout, self.max_retries
+            )
+        return self._aclient
+
+
+class ScrapeInput(BaseModel):
+    url: str = Field(description="The absolute URL of the page to read.")
+
+
+class HydrafetchScrapeTool(_HydrafetchTool):
+    """Read one web page and return it as clean markdown."""
+
+    name: str = "hydrafetch_scrape"
+    description: str = (
+        "Read a single web page and return its main content as clean markdown. "
+        "Use this when you already have the URL of the page you need."
+    )
+    args_schema: type[BaseModel] = ScrapeInput
+
+    def _run(self, url: str, run_manager: CallbackManagerForToolRun | None = None) -> str:
+        data = unwrap(self._sync().scrape(url, formats=["markdown"]))
+        return str(data.get("markdown") or "")
+
+    async def _arun(
+        self, url: str, run_manager: AsyncCallbackManagerForToolRun | None = None
+    ) -> str:
+        data = unwrap(await self._async().scrape(url, formats=["markdown"]))
+        return str(data.get("markdown") or "")
+
+
+class SearchInput(BaseModel):
+    query: str = Field(description="What to search the web for.")
+    limit: int = Field(default=5, description="How many results to return.")
+
+
+class HydrafetchSearchTool(_HydrafetchTool):
+    """Search the web and return titles, URLs and snippets."""
+
+    name: str = "hydrafetch_search"
+    description: str = (
+        "Search the web and return a list of results with title, url and snippet. "
+        "Use this when you need to find pages rather than read a known URL."
+    )
+    args_schema: type[BaseModel] = SearchInput
+
+    def _run(
+        self, query: str, limit: int = 5, run_manager: CallbackManagerForToolRun | None = None
+    ) -> str:
+        data = unwrap(self._sync().search(query, limit=limit))
+        return _dump(_trim_results(data, limit))
+
+    async def _arun(
+        self,
+        query: str,
+        limit: int = 5,
+        run_manager: AsyncCallbackManagerForToolRun | None = None,
+    ) -> str:
+        data = unwrap(await self._async().search(query, limit=limit))
+        return _dump(_trim_results(data, limit))
+
+
+class ExtractInput(BaseModel):
+    urls: list[str] = Field(description="Absolute URLs to extract from.")
+    json_schema: dict[str, Any] | None = Field(
+        default=None, description="JSON Schema describing the shape to extract."
+    )
+    prompt: str | None = Field(
+        default=None, description="Plain-language description of what to extract."
+    )
+
+
+class HydrafetchExtractTool(_HydrafetchTool):
+    """Extract structured JSON from one or more pages."""
+
+    name: str = "hydrafetch_extract"
+    description: str = (
+        "Extract structured JSON from one or more web pages, either against a JSON Schema "
+        "or from a plain-language description of the fields you want."
+    )
+    args_schema: type[BaseModel] = ExtractInput
+
+    def _run(
+        self,
+        urls: list[str],
+        json_schema: dict[str, Any] | None = None,
+        prompt: str | None = None,
+        run_manager: CallbackManagerForToolRun | None = None,
+    ) -> str:
+        return _dump(unwrap(self._sync().extract(urls, **_extract_options(json_schema, prompt))))
+
+    async def _arun(
+        self,
+        urls: list[str],
+        json_schema: dict[str, Any] | None = None,
+        prompt: str | None = None,
+        run_manager: AsyncCallbackManagerForToolRun | None = None,
+    ) -> str:
+        payload = await self._async().extract(urls, **_extract_options(json_schema, prompt))
+        return _dump(unwrap(payload))
+
+
+class BrandInput(BaseModel):
+    domain: str = Field(description="A company domain, for example 'stripe.com'.")
+
+
+class HydrafetchBrandTool(_HydrafetchTool):
+    """Resolve a company domain to its brand and firmographic record."""
+
+    name: str = "hydrafetch_brand"
+    description: str = (
+        "Given a company domain, return its brand record: name, description, logo, "
+        "colours, and social profiles. Use this to enrich a company from its website."
+    )
+    args_schema: type[BaseModel] = BrandInput
+
+    def _run(self, domain: str, run_manager: CallbackManagerForToolRun | None = None) -> str:
+        return _dump(unwrap(self._sync().brand(domain)))
+
+    async def _arun(
+        self, domain: str, run_manager: AsyncCallbackManagerForToolRun | None = None
+    ) -> str:
+        return _dump(unwrap(await self._async().brand(domain)))
+
+
+def _extract_options(json_schema: dict[str, Any] | None, prompt: str | None) -> dict[str, Any]:
+    options: dict[str, Any] = {}
+    if json_schema is not None:
+        options["schema"] = json_schema
+    if prompt is not None:
+        options["prompt"] = prompt
+    return options
+
+
+def _trim_results(data: dict[str, Any], limit: int) -> dict[str, Any]:
+    results = [item for item in (data.get("results") or []) if isinstance(item, dict)]
+    return {
+        "query": data.get("query"),
+        "results": [
+            {k: item.get(k) for k in ("title", "url", "snippet") if item.get(k) is not None}
+            for item in results[:limit]
+        ],
+    }
+
+
+def _dump(payload: Any) -> str:
+    return json.dumps(payload, ensure_ascii=False, default=str)

langchain_hydrafetch-0.1.0.dist-info/METADATA

@@ -0,0 +1,264 @@
+Metadata-Version: 2.5
+Name: langchain-hydrafetch
+Version: 0.1.0
+Summary: LangChain integration for Hydrafetch. Load, search and extract clean web data for agents and RAG.
+Project-URL: Homepage, https://hydrafetch.com
+Project-URL: Documentation, https://docs.hydrafetch.com
+Project-URL: Source, https://github.com/Hydrafetch/langchain-hydrafetch
+Project-URL: Issues, https://github.com/Hydrafetch/langchain-hydrafetch/issues
+Author-email: Hydrafetch <team@hydrafetch.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agents,document-loader,hydrafetch,langchain,llm,markdown,rag,web-scraping
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: hydrafetch>=0.1.1
+Requires-Dist: langchain-core<2,>=1.0
+Requires-Dist: pydantic>=2
+Description-Content-Type: text/markdown
+
+# langchain-hydrafetch
+
+[![PyPI](https://img.shields.io/pypi/v/langchain-hydrafetch)](https://pypi.org/project/langchain-hydrafetch/)
+[![CI](https://github.com/Hydrafetch/langchain-hydrafetch/actions/workflows/ci.yml/badge.svg)](https://github.com/Hydrafetch/langchain-hydrafetch/actions/workflows/ci.yml)
+[![Python](https://img.shields.io/pypi/pyversions/langchain-hydrafetch)](https://pypi.org/project/langchain-hydrafetch/)
+
+LangChain integration for the [Hydrafetch](https://hydrafetch.com) web data API.
+
+Load pages as documents, search the web as a retriever, and give agents four tools that read, extract and enrich from the live web. Sync and async, fully typed.
+
+## Installation
+
+```bash
+pip install langchain-hydrafetch
+```
+
+Create a key at [app.hydrafetch.com](https://app.hydrafetch.com) and set it:
+
+```bash
+export HYDRAFETCH_API_KEY="hf_..."
+```
+
+Everything below reads that variable when no `api_key=` is passed.
+
+## Quick start
+
+```python
+from langchain_hydrafetch import HydrafetchLoader
+
+docs = HydrafetchLoader("https://example.com/article").load()
+print(docs[0].page_content)
+print(docs[0].metadata["title"])
+```
+
+## Document loader
+
+Three modes. `scrape` loads one page, `crawl` walks a site and loads every page it finds, and `map` returns one document per discovered URL without fetching the bodies.
+
+```python
+HydrafetchLoader("https://example.com/article").load()
+HydrafetchLoader("https://example.com", mode="crawl", params={"limit": 50}).load()
+HydrafetchLoader("https://example.com", mode="map").load()
+```
+
+`content_format` chooses what lands in `page_content` — `markdown` by default, or `html`, `rawHtml`, `summary`. `params` is forwarded to the API untouched, so anything the endpoint accepts works here:
+
+```python
+HydrafetchLoader(
+    "https://example.com/article",
+    content_format="markdown",
+    params={"onlyMainContent": True, "preferStructure": True, "blockAds": True},
+).load()
+```
+
+Documents carry `source` and `status`, plus whatever page metadata was found: `title`, `description`, `language`, `site_name`, `author`, `published_time`, `word_count`, `page_type`, `image`. A redirect adds `final_url`; a crawled page adds `depth`; a mapped URL adds `lastmod` when the sitemap declares one.
+
+Use `lazy_load()` to stream a large crawl instead of building the whole list in memory:
+
+```python
+for doc in HydrafetchLoader("https://example.com", mode="crawl").lazy_load():
+    index.add(doc)
+```
+
+### Error pages are refused, not loaded
+
+A URL that answers with an error status raises instead of returning a document, because the body of a 404 page is not the page you asked for and a retrieval index should not quietly absorb one.
+
+```python
+HydrafetchLoader("https://example.com/gone").load()
+# ValueError: https://example.com/gone returned HTTP 404. The body of an error
+# page is not the page you asked for; pass raise_for_status=False to load it anyway.
+```
+
+In `crawl` mode a single dead page must not throw away the whole job, so error pages are dropped and the rest of the crawl is kept. Pass `raise_for_status=False` to load error pages in either mode.
+
+## Retriever
+
+```python
+from langchain_hydrafetch import HydrafetchSearchRetriever
+
+retriever = HydrafetchSearchRetriever(k=5)
+docs = retriever.invoke("best open source vector databases")
+```
+
+Each document holds the result snippet, with `source`, `title` and `rank` in metadata. Pass `scrape_content=True` to fetch and return the full page body for every result instead:
+
+```python
+HydrafetchSearchRetriever(k=3, scrape_content=True).invoke("...")
+```
+
+That costs one extra credit per result. `k` works as a constructor argument or per call, and `search_params` is forwarded to the search endpoint:
+
+```python
+retriever.invoke("...", k=2)
+HydrafetchSearchRetriever(search_params={"country": "us"})
+```
+
+## Agent tools
+
+```python
+from langchain_hydrafetch import (
+    HydrafetchBrandTool,
+    HydrafetchExtractTool,
+    HydrafetchScrapeTool,
+    HydrafetchSearchTool,
+)
+
+tools = [
+    HydrafetchSearchTool(),
+    HydrafetchScrapeTool(),
+    HydrafetchExtractTool(),
+    HydrafetchBrandTool(),
+]
+```
+
+| tool | argument | returns |
+| --- | --- | --- |
+| `hydrafetch_search` | `query`, `limit` | JSON with `query` and `results` of title, url, snippet |
+| `hydrafetch_scrape` | `url` | the page as markdown |
+| `hydrafetch_extract` | `urls`, `json_schema` or `prompt` | JSON with one entry per URL |
+| `hydrafetch_brand` | `domain` | JSON brand record: name, description, tagline, logo assets, colours, fonts, socials |
+
+Search finds pages, scrape reads a page you already have. Giving an agent both is the usual setup.
+
+Structured extraction takes either a schema or a plain-language description:
+
+```python
+HydrafetchExtractTool().invoke({
+    "urls": ["https://example.com/pricing"],
+    "json_schema": {
+        "type": "object",
+        "properties": {"plans": {"type": "array", "items": {"type": "string"}}},
+    },
+})
+
+HydrafetchExtractTool().invoke({
+    "urls": ["https://example.com/about"],
+    "prompt": "the founding year and the headquarters city",
+})
+```
+
+## Async
+
+The retriever and every tool support `ainvoke`, and the loader supports `alazy_load` and `aload`:
+
+```python
+docs = await HydrafetchSearchRetriever().ainvoke("...")
+text = await HydrafetchScrapeTool().ainvoke({"url": "https://example.com"})
+```
+
+## Error handling
+
+Failures raise `HydrafetchError` from the underlying client, carrying the API's error code, HTTP status and request id.
+
+```python
+from hydrafetch import HydrafetchError, HydrafetchTimeout
+
+try:
+    docs = HydrafetchLoader(url).load()
+except HydrafetchTimeout:
+    raise
+except HydrafetchError as err:
+    if err.is_out_of_credits:
+        top_up()
+    elif err.is_retryable:
+        enqueue(url)
+    else:
+        raise
+```
+
+| Status | Meaning | Retried |
+| --- | --- | --- |
+| 400, 422 | invalid request | no |
+| 401, 403 | invalid or missing key | no |
+| 402 | out of credits | no |
+| 429 | rate limited | yes, twice with backoff |
+| 5xx | upstream failure | yes, twice with backoff |
+
+A page that loads but answers with an error status raises `ValueError` from the loader instead — that is a bad URL, not a failed request.
+
+## Configuration
+
+Every class accepts the same connection options, all optional:
+
+| option | default | meaning |
+| --- | --- | --- |
+| `api_key` | `HYDRAFETCH_API_KEY` | your API key |
+| `base_url` | `https://api.hydrafetch.com` | API base URL |
+| `timeout` | `120.0` | per-request timeout in seconds |
+| `max_retries` | `2` | retries on 429 and 5xx |
+
+## Credits
+
+| call | credits |
+| --- | --- |
+| loader, `scrape` mode | 1 |
+| loader, `map` mode | 1 |
+| loader, `crawl` mode | 1 per page |
+| retriever | 1, plus 1 per result with `scrape_content=True` |
+| `hydrafetch_scrape` | 1 |
+| `hydrafetch_search` | 1 |
+| `hydrafetch_extract` | 5 per URL |
+| `hydrafetch_brand` | 5 |
+
+Failed requests are not billed. Pricing does not vary with page difficulty, so there is no render, stealth or proxy option to set.
+
+## Implementation notes
+
+- Prefer `map` then a batch of `scrape` loads over a broad `crawl`. Fetching a whole site and discarding most of it is the most common source of wasted credits.
+- `params` in `crawl` mode is forwarded to the crawl endpoint, so per-page options belong under `scrapeOptions`. At the top level they are ignored.
+- `preferStructure` is off by default. Turn it on when headings, lists and tables matter; leave it off for raw article text.
+- The loader constructs its client eagerly, so a missing API key fails at construction rather than at `load()`.
+- The retriever and tools construct their clients lazily on first use, which keeps them cheap to build and safe to define at import time.
+- Metadata keys are snake_cased on the way out, so the API's `siteName` becomes `site_name`.
+- Scraped content is untrusted input. Do not pass it to a model as instructions, and keep `metadata["source"]` with anything extracted from it.
+
+## Development
+
+```bash
+uv sync
+uv run ruff check src tests
+uv run ruff format --check src tests
+uv run pytest -q
+```
+
+Unit tests run offline against fake clients. The integration tests are LangChain's own `langchain-tests` standard suite; they are skipped unless `HYDRAFETCH_API_KEY` is set, and they spend real credits.
+
+## Links
+
+- [Documentation](https://docs.hydrafetch.com)
+- [OpenAPI specification](https://api.hydrafetch.com/openapi.json)
+- [MCP server and editor setup](https://hydrafetch.com/mcp)
+- [Python client](https://github.com/Hydrafetch/python-sdk) — the client this integration wraps
+- [LlamaIndex reader](https://github.com/Hydrafetch/llama-index-readers-hydrafetch)
+- Other clients: [Node](https://github.com/Hydrafetch/node-sdk) · [Go](https://github.com/Hydrafetch/go-sdk) · [Ruby](https://github.com/Hydrafetch/ruby-sdk) · [Rust](https://github.com/Hydrafetch/rust-sdk) · [PHP](https://github.com/Hydrafetch/php-sdk)
+
+## License
+
+MIT

langchain_hydrafetch-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+langchain_hydrafetch/__init__.py,sha256=nyiv9PNoZHeacnBQ9zW6lgshGjnLNkWEO3iEd43rgng,492
+langchain_hydrafetch/_common.py,sha256=h7XGXDqE9WQSun3LSJokT-y45M259S_nuiDprwLsHrA,2683
+langchain_hydrafetch/document_loaders.py,sha256=Or5ukoCTB5ki_yYQI40wN1uuxhHcxhssKz9J5uAYdw4,3544
+langchain_hydrafetch/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+langchain_hydrafetch/retrievers.py,sha256=ZRFL99hG53ZA8_EbtR8UCmvkc3cPdtY6s24a34F-eUo,3882
+langchain_hydrafetch/tools.py,sha256=BuLYa3ZTgrQy8dmrDHLmUGaLvmURY9_5EdEiv8570GU,6137
+langchain_hydrafetch-0.1.0.dist-info/METADATA,sha256=0VwuJaWgUPRJwDfPpSO0AEp77AiemJdcYPUXgTVFwXk,9992
+langchain_hydrafetch-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+langchain_hydrafetch-0.1.0.dist-info/licenses/LICENSE,sha256=yEyqX8ttI5_YWUzo5l0UqsOSYz5ujIKPeLQsq8ME3NE,1067
+langchain_hydrafetch-0.1.0.dist-info/RECORD,,

langchain_hydrafetch-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

langchain_hydrafetch-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2026 Hydrafetch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.