langchain-hydrafetch 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langchain_hydrafetch-0.1.0/.github/workflows/ci.yml +20 -0
- langchain_hydrafetch-0.1.0/.github/workflows/publish.yml +27 -0
- langchain_hydrafetch-0.1.0/.gitignore +9 -0
- langchain_hydrafetch-0.1.0/LICENSE +9 -0
- langchain_hydrafetch-0.1.0/PKG-INFO +264 -0
- langchain_hydrafetch-0.1.0/README.md +239 -0
- langchain_hydrafetch-0.1.0/pyproject.toml +46 -0
- langchain_hydrafetch-0.1.0/src/langchain_hydrafetch/__init__.py +20 -0
- langchain_hydrafetch-0.1.0/src/langchain_hydrafetch/_common.py +93 -0
- langchain_hydrafetch-0.1.0/src/langchain_hydrafetch/document_loaders.py +98 -0
- langchain_hydrafetch-0.1.0/src/langchain_hydrafetch/py.typed +0 -0
- langchain_hydrafetch-0.1.0/src/langchain_hydrafetch/retrievers.py +110 -0
- langchain_hydrafetch-0.1.0/src/langchain_hydrafetch/tools.py +180 -0
- langchain_hydrafetch-0.1.0/tests/__init__.py +0 -0
- langchain_hydrafetch-0.1.0/tests/integration_tests/__init__.py +0 -0
- langchain_hydrafetch-0.1.0/tests/integration_tests/test_tools_standard.py +34 -0
- langchain_hydrafetch-0.1.0/tests/unit_tests/__init__.py +0 -0
- langchain_hydrafetch-0.1.0/tests/unit_tests/conftest.py +8 -0
- langchain_hydrafetch-0.1.0/tests/unit_tests/test_loader.py +179 -0
- langchain_hydrafetch-0.1.0/tests/unit_tests/test_retriever.py +76 -0
- langchain_hydrafetch-0.1.0/tests/unit_tests/test_tools_standard.py +68 -0
- langchain_hydrafetch-0.1.0/uv.lock +2008 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
matrix:
|
|
13
|
+
python: ['3.10', '3.11', '3.12', '3.13']
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: astral-sh/setup-uv@v5
|
|
17
|
+
- run: uv sync --python ${{ matrix.python }}
|
|
18
|
+
- run: uv run ruff check src tests
|
|
19
|
+
- run: uv run ruff format --check src tests
|
|
20
|
+
- run: uv run pytest -q tests/unit_tests
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
name: Publish
|
|
2
|
+
|
|
3
|
+
# Trusted publishing to PyPI: no API token stored anywhere, the id-token proves
|
|
4
|
+
# the release came from this workflow in this repo.
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
tags: ['v*']
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
publish:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
environment: pypi
|
|
13
|
+
permissions:
|
|
14
|
+
contents: read
|
|
15
|
+
id-token: write
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
- uses: astral-sh/setup-uv@v5
|
|
19
|
+
- run: uv sync
|
|
20
|
+
- run: uv run pytest -q tests/unit_tests
|
|
21
|
+
- name: Fail if the tag and pyproject disagree
|
|
22
|
+
run: |
|
|
23
|
+
TAG="${GITHUB_REF_NAME#v}"
|
|
24
|
+
PKG=$(uv run python -c "import tomllib;print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
|
|
25
|
+
[ "$TAG" = "$PKG" ] || { echo "tag $TAG != pyproject $PKG"; exit 1; }
|
|
26
|
+
- run: uv build
|
|
27
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hydrafetch
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
6
|
+
|
|
7
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: langchain-hydrafetch
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: LangChain integration for Hydrafetch. Load, search and extract clean web data for agents and RAG.
|
|
5
|
+
Project-URL: Homepage, https://hydrafetch.com
|
|
6
|
+
Project-URL: Documentation, https://docs.hydrafetch.com
|
|
7
|
+
Project-URL: Source, https://github.com/Hydrafetch/langchain-hydrafetch
|
|
8
|
+
Project-URL: Issues, https://github.com/Hydrafetch/langchain-hydrafetch/issues
|
|
9
|
+
Author-email: Hydrafetch <team@hydrafetch.com>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agents,document-loader,hydrafetch,langchain,llm,markdown,rag,web-scraping
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Requires-Dist: hydrafetch>=0.1.1
|
|
22
|
+
Requires-Dist: langchain-core<2,>=1.0
|
|
23
|
+
Requires-Dist: pydantic>=2
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# langchain-hydrafetch
|
|
27
|
+
|
|
28
|
+
[](https://pypi.org/project/langchain-hydrafetch/)
|
|
29
|
+
[](https://github.com/Hydrafetch/langchain-hydrafetch/actions/workflows/ci.yml)
|
|
30
|
+
[](https://pypi.org/project/langchain-hydrafetch/)
|
|
31
|
+
|
|
32
|
+
LangChain integration for the [Hydrafetch](https://hydrafetch.com) web data API.
|
|
33
|
+
|
|
34
|
+
Load pages as documents, search the web as a retriever, and give agents four tools that read, extract and enrich from the live web. Sync and async, fully typed.
|
|
35
|
+
|
|
36
|
+
## Installation
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install langchain-hydrafetch
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Create a key at [app.hydrafetch.com](https://app.hydrafetch.com) and set it:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
export HYDRAFETCH_API_KEY="hf_..."
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Everything below reads that variable when no `api_key=` is passed.
|
|
49
|
+
|
|
50
|
+
## Quick start
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from langchain_hydrafetch import HydrafetchLoader
|
|
54
|
+
|
|
55
|
+
docs = HydrafetchLoader("https://example.com/article").load()
|
|
56
|
+
print(docs[0].page_content)
|
|
57
|
+
print(docs[0].metadata["title"])
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Document loader
|
|
61
|
+
|
|
62
|
+
Three modes. `scrape` loads one page, `crawl` walks a site and loads every page it finds, and `map` returns one document per discovered URL without fetching the bodies.
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
HydrafetchLoader("https://example.com/article").load()
|
|
66
|
+
HydrafetchLoader("https://example.com", mode="crawl", params={"limit": 50}).load()
|
|
67
|
+
HydrafetchLoader("https://example.com", mode="map").load()
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`content_format` chooses what lands in `page_content` — `markdown` by default, or `html`, `rawHtml`, `summary`. `params` is forwarded to the API untouched, so anything the endpoint accepts works here:
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
HydrafetchLoader(
|
|
74
|
+
"https://example.com/article",
|
|
75
|
+
content_format="markdown",
|
|
76
|
+
params={"onlyMainContent": True, "preferStructure": True, "blockAds": True},
|
|
77
|
+
).load()
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Documents carry `source` and `status`, plus whatever page metadata was found: `title`, `description`, `language`, `site_name`, `author`, `published_time`, `word_count`, `page_type`, `image`. A redirect adds `final_url`; a crawled page adds `depth`; a mapped URL adds `lastmod` when the sitemap declares one.
|
|
81
|
+
|
|
82
|
+
Use `lazy_load()` to stream a large crawl instead of building the whole list in memory:
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
for doc in HydrafetchLoader("https://example.com", mode="crawl").lazy_load():
|
|
86
|
+
index.add(doc)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### Error pages are refused, not loaded
|
|
90
|
+
|
|
91
|
+
A URL that answers with an error status raises instead of returning a document, because the body of a 404 page is not the page you asked for and a retrieval index should not quietly absorb one.
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
HydrafetchLoader("https://example.com/gone").load()
|
|
95
|
+
# ValueError: https://example.com/gone returned HTTP 404. The body of an error
|
|
96
|
+
# page is not the page you asked for; pass raise_for_status=False to load it anyway.
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
In `crawl` mode a single dead page must not throw away the whole job, so error pages are dropped and the rest of the crawl is kept. Pass `raise_for_status=False` to load error pages in either mode.
|
|
100
|
+
|
|
101
|
+
## Retriever
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
from langchain_hydrafetch import HydrafetchSearchRetriever
|
|
105
|
+
|
|
106
|
+
retriever = HydrafetchSearchRetriever(k=5)
|
|
107
|
+
docs = retriever.invoke("best open source vector databases")
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Each document holds the result snippet, with `source`, `title` and `rank` in metadata. Pass `scrape_content=True` to fetch and return the full page body for every result instead:
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
HydrafetchSearchRetriever(k=3, scrape_content=True).invoke("...")
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
That costs one extra credit per result. `k` works as a constructor argument or per call, and `search_params` is forwarded to the search endpoint:
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
retriever.invoke("...", k=2)
|
|
120
|
+
HydrafetchSearchRetriever(search_params={"country": "us"})
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Agent tools
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
from langchain_hydrafetch import (
|
|
127
|
+
HydrafetchBrandTool,
|
|
128
|
+
HydrafetchExtractTool,
|
|
129
|
+
HydrafetchScrapeTool,
|
|
130
|
+
HydrafetchSearchTool,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
tools = [
|
|
134
|
+
HydrafetchSearchTool(),
|
|
135
|
+
HydrafetchScrapeTool(),
|
|
136
|
+
HydrafetchExtractTool(),
|
|
137
|
+
HydrafetchBrandTool(),
|
|
138
|
+
]
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
| tool | argument | returns |
|
|
142
|
+
| --- | --- | --- |
|
|
143
|
+
| `hydrafetch_search` | `query`, `limit` | JSON with `query` and `results` of title, url, snippet |
|
|
144
|
+
| `hydrafetch_scrape` | `url` | the page as markdown |
|
|
145
|
+
| `hydrafetch_extract` | `urls`, `json_schema` or `prompt` | JSON with one entry per URL |
|
|
146
|
+
| `hydrafetch_brand` | `domain` | JSON brand record: name, description, tagline, logo assets, colours, fonts, socials |
|
|
147
|
+
|
|
148
|
+
Search finds pages, scrape reads a page you already have. Giving an agent both is the usual setup.
|
|
149
|
+
|
|
150
|
+
Structured extraction takes either a schema or a plain-language description:
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
HydrafetchExtractTool().invoke({
|
|
154
|
+
"urls": ["https://example.com/pricing"],
|
|
155
|
+
"json_schema": {
|
|
156
|
+
"type": "object",
|
|
157
|
+
"properties": {"plans": {"type": "array", "items": {"type": "string"}}},
|
|
158
|
+
},
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
HydrafetchExtractTool().invoke({
|
|
162
|
+
"urls": ["https://example.com/about"],
|
|
163
|
+
"prompt": "the founding year and the headquarters city",
|
|
164
|
+
})
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
## Async
|
|
168
|
+
|
|
169
|
+
The retriever and every tool support `ainvoke`, and the loader supports `alazy_load` and `aload`:
|
|
170
|
+
|
|
171
|
+
```python
|
|
172
|
+
docs = await HydrafetchSearchRetriever().ainvoke("...")
|
|
173
|
+
text = await HydrafetchScrapeTool().ainvoke({"url": "https://example.com"})
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
## Error handling
|
|
177
|
+
|
|
178
|
+
Failures raise `HydrafetchError` from the underlying client, carrying the API's error code, HTTP status and request id.
|
|
179
|
+
|
|
180
|
+
```python
|
|
181
|
+
from hydrafetch import HydrafetchError, HydrafetchTimeout
|
|
182
|
+
|
|
183
|
+
try:
|
|
184
|
+
docs = HydrafetchLoader(url).load()
|
|
185
|
+
except HydrafetchTimeout:
|
|
186
|
+
raise
|
|
187
|
+
except HydrafetchError as err:
|
|
188
|
+
if err.is_out_of_credits:
|
|
189
|
+
top_up()
|
|
190
|
+
elif err.is_retryable:
|
|
191
|
+
enqueue(url)
|
|
192
|
+
else:
|
|
193
|
+
raise
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
| Status | Meaning | Retried |
|
|
197
|
+
| --- | --- | --- |
|
|
198
|
+
| 400, 422 | invalid request | no |
|
|
199
|
+
| 401, 403 | invalid or missing key | no |
|
|
200
|
+
| 402 | out of credits | no |
|
|
201
|
+
| 429 | rate limited | yes, twice with backoff |
|
|
202
|
+
| 5xx | upstream failure | yes, twice with backoff |
|
|
203
|
+
|
|
204
|
+
A page that loads but answers with an error status raises `ValueError` from the loader instead — that is a bad URL, not a failed request.
|
|
205
|
+
|
|
206
|
+
## Configuration
|
|
207
|
+
|
|
208
|
+
Every class accepts the same connection options, all optional:
|
|
209
|
+
|
|
210
|
+
| option | default | meaning |
|
|
211
|
+
| --- | --- | --- |
|
|
212
|
+
| `api_key` | `HYDRAFETCH_API_KEY` | your API key |
|
|
213
|
+
| `base_url` | `https://api.hydrafetch.com` | API base URL |
|
|
214
|
+
| `timeout` | `120.0` | per-request timeout in seconds |
|
|
215
|
+
| `max_retries` | `2` | retries on 429 and 5xx |
|
|
216
|
+
|
|
217
|
+
## Credits
|
|
218
|
+
|
|
219
|
+
| call | credits |
|
|
220
|
+
| --- | --- |
|
|
221
|
+
| loader, `scrape` mode | 1 |
|
|
222
|
+
| loader, `map` mode | 1 |
|
|
223
|
+
| loader, `crawl` mode | 1 per page |
|
|
224
|
+
| retriever | 1, plus 1 per result with `scrape_content=True` |
|
|
225
|
+
| `hydrafetch_scrape` | 1 |
|
|
226
|
+
| `hydrafetch_search` | 1 |
|
|
227
|
+
| `hydrafetch_extract` | 5 per URL |
|
|
228
|
+
| `hydrafetch_brand` | 5 |
|
|
229
|
+
|
|
230
|
+
Failed requests are not billed. Pricing does not vary with page difficulty, so there is no render, stealth or proxy option to set.
|
|
231
|
+
|
|
232
|
+
## Implementation notes
|
|
233
|
+
|
|
234
|
+
- Prefer `map` then a batch of `scrape` loads over a broad `crawl`. Fetching a whole site and discarding most of it is the most common source of wasted credits.
|
|
235
|
+
- `params` in `crawl` mode is forwarded to the crawl endpoint, so per-page options belong under `scrapeOptions`. At the top level they are ignored.
|
|
236
|
+
- `preferStructure` is off by default. Turn it on when headings, lists and tables matter; leave it off for raw article text.
|
|
237
|
+
- The loader constructs its client eagerly, so a missing API key fails at construction rather than at `load()`.
|
|
238
|
+
- The retriever and tools construct their clients lazily on first use, which keeps them cheap to build and safe to define at import time.
|
|
239
|
+
- Metadata keys are snake_cased on the way out, so the API's `siteName` becomes `site_name`.
|
|
240
|
+
- Scraped content is untrusted input. Do not pass it to a model as instructions, and keep `metadata["source"]` with anything extracted from it.
|
|
241
|
+
|
|
242
|
+
## Development
|
|
243
|
+
|
|
244
|
+
```bash
|
|
245
|
+
uv sync
|
|
246
|
+
uv run ruff check src tests
|
|
247
|
+
uv run ruff format --check src tests
|
|
248
|
+
uv run pytest -q
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
Unit tests run offline against fake clients. The integration tests are LangChain's own `langchain-tests` standard suite; they are skipped unless `HYDRAFETCH_API_KEY` is set, and they spend real credits.
|
|
252
|
+
|
|
253
|
+
## Links
|
|
254
|
+
|
|
255
|
+
- [Documentation](https://docs.hydrafetch.com)
|
|
256
|
+
- [OpenAPI specification](https://api.hydrafetch.com/openapi.json)
|
|
257
|
+
- [MCP server and editor setup](https://hydrafetch.com/mcp)
|
|
258
|
+
- [Python client](https://github.com/Hydrafetch/python-sdk) — the client this integration wraps
|
|
259
|
+
- [LlamaIndex reader](https://github.com/Hydrafetch/llama-index-readers-hydrafetch)
|
|
260
|
+
- Other clients: [Node](https://github.com/Hydrafetch/node-sdk) · [Go](https://github.com/Hydrafetch/go-sdk) · [Ruby](https://github.com/Hydrafetch/ruby-sdk) · [Rust](https://github.com/Hydrafetch/rust-sdk) · [PHP](https://github.com/Hydrafetch/php-sdk)
|
|
261
|
+
|
|
262
|
+
## License
|
|
263
|
+
|
|
264
|
+
MIT
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
# langchain-hydrafetch
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/langchain-hydrafetch/)
|
|
4
|
+
[](https://github.com/Hydrafetch/langchain-hydrafetch/actions/workflows/ci.yml)
|
|
5
|
+
[](https://pypi.org/project/langchain-hydrafetch/)
|
|
6
|
+
|
|
7
|
+
LangChain integration for the [Hydrafetch](https://hydrafetch.com) web data API.
|
|
8
|
+
|
|
9
|
+
Load pages as documents, search the web as a retriever, and give agents four tools that read, extract and enrich from the live web. Sync and async, fully typed.
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install langchain-hydrafetch
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Create a key at [app.hydrafetch.com](https://app.hydrafetch.com) and set it:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
export HYDRAFETCH_API_KEY="hf_..."
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Everything below reads that variable when no `api_key=` is passed.
|
|
24
|
+
|
|
25
|
+
## Quick start
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
from langchain_hydrafetch import HydrafetchLoader
|
|
29
|
+
|
|
30
|
+
docs = HydrafetchLoader("https://example.com/article").load()
|
|
31
|
+
print(docs[0].page_content)
|
|
32
|
+
print(docs[0].metadata["title"])
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Document loader
|
|
36
|
+
|
|
37
|
+
Three modes. `scrape` loads one page, `crawl` walks a site and loads every page it finds, and `map` returns one document per discovered URL without fetching the bodies.
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
HydrafetchLoader("https://example.com/article").load()
|
|
41
|
+
HydrafetchLoader("https://example.com", mode="crawl", params={"limit": 50}).load()
|
|
42
|
+
HydrafetchLoader("https://example.com", mode="map").load()
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
`content_format` chooses what lands in `page_content` — `markdown` by default, or `html`, `rawHtml`, `summary`. `params` is forwarded to the API untouched, so anything the endpoint accepts works here:
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
HydrafetchLoader(
|
|
49
|
+
"https://example.com/article",
|
|
50
|
+
content_format="markdown",
|
|
51
|
+
params={"onlyMainContent": True, "preferStructure": True, "blockAds": True},
|
|
52
|
+
).load()
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Documents carry `source` and `status`, plus whatever page metadata was found: `title`, `description`, `language`, `site_name`, `author`, `published_time`, `word_count`, `page_type`, `image`. A redirect adds `final_url`; a crawled page adds `depth`; a mapped URL adds `lastmod` when the sitemap declares one.
|
|
56
|
+
|
|
57
|
+
Use `lazy_load()` to stream a large crawl instead of building the whole list in memory:
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
for doc in HydrafetchLoader("https://example.com", mode="crawl").lazy_load():
|
|
61
|
+
index.add(doc)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
### Error pages are refused, not loaded
|
|
65
|
+
|
|
66
|
+
A URL that answers with an error status raises instead of returning a document, because the body of a 404 page is not the page you asked for and a retrieval index should not quietly absorb one.
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
HydrafetchLoader("https://example.com/gone").load()
|
|
70
|
+
# ValueError: https://example.com/gone returned HTTP 404. The body of an error
|
|
71
|
+
# page is not the page you asked for; pass raise_for_status=False to load it anyway.
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
In `crawl` mode a single dead page must not throw away the whole job, so error pages are dropped and the rest of the crawl is kept. Pass `raise_for_status=False` to load error pages in either mode.
|
|
75
|
+
|
|
76
|
+
## Retriever
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from langchain_hydrafetch import HydrafetchSearchRetriever
|
|
80
|
+
|
|
81
|
+
retriever = HydrafetchSearchRetriever(k=5)
|
|
82
|
+
docs = retriever.invoke("best open source vector databases")
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Each document holds the result snippet, with `source`, `title` and `rank` in metadata. Pass `scrape_content=True` to fetch and return the full page body for every result instead:
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
HydrafetchSearchRetriever(k=3, scrape_content=True).invoke("...")
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
That costs one extra credit per result. `k` works as a constructor argument or per call, and `search_params` is forwarded to the search endpoint:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
retriever.invoke("...", k=2)
|
|
95
|
+
HydrafetchSearchRetriever(search_params={"country": "us"})
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
## Agent tools
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from langchain_hydrafetch import (
|
|
102
|
+
HydrafetchBrandTool,
|
|
103
|
+
HydrafetchExtractTool,
|
|
104
|
+
HydrafetchScrapeTool,
|
|
105
|
+
HydrafetchSearchTool,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
tools = [
|
|
109
|
+
HydrafetchSearchTool(),
|
|
110
|
+
HydrafetchScrapeTool(),
|
|
111
|
+
HydrafetchExtractTool(),
|
|
112
|
+
HydrafetchBrandTool(),
|
|
113
|
+
]
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
| tool | argument | returns |
|
|
117
|
+
| --- | --- | --- |
|
|
118
|
+
| `hydrafetch_search` | `query`, `limit` | JSON with `query` and `results` of title, url, snippet |
|
|
119
|
+
| `hydrafetch_scrape` | `url` | the page as markdown |
|
|
120
|
+
| `hydrafetch_extract` | `urls`, `json_schema` or `prompt` | JSON with one entry per URL |
|
|
121
|
+
| `hydrafetch_brand` | `domain` | JSON brand record: name, description, tagline, logo assets, colours, fonts, socials |
|
|
122
|
+
|
|
123
|
+
Search finds pages, scrape reads a page you already have. Giving an agent both is the usual setup.
|
|
124
|
+
|
|
125
|
+
Structured extraction takes either a schema or a plain-language description:
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
HydrafetchExtractTool().invoke({
|
|
129
|
+
"urls": ["https://example.com/pricing"],
|
|
130
|
+
"json_schema": {
|
|
131
|
+
"type": "object",
|
|
132
|
+
"properties": {"plans": {"type": "array", "items": {"type": "string"}}},
|
|
133
|
+
},
|
|
134
|
+
})
|
|
135
|
+
|
|
136
|
+
HydrafetchExtractTool().invoke({
|
|
137
|
+
"urls": ["https://example.com/about"],
|
|
138
|
+
"prompt": "the founding year and the headquarters city",
|
|
139
|
+
})
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Async
|
|
143
|
+
|
|
144
|
+
The retriever and every tool support `ainvoke`, and the loader supports `alazy_load` and `aload`:
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
docs = await HydrafetchSearchRetriever().ainvoke("...")
|
|
148
|
+
text = await HydrafetchScrapeTool().ainvoke({"url": "https://example.com"})
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
## Error handling
|
|
152
|
+
|
|
153
|
+
Failures raise `HydrafetchError` from the underlying client, carrying the API's error code, HTTP status and request id.
|
|
154
|
+
|
|
155
|
+
```python
|
|
156
|
+
from hydrafetch import HydrafetchError, HydrafetchTimeout
|
|
157
|
+
|
|
158
|
+
try:
|
|
159
|
+
docs = HydrafetchLoader(url).load()
|
|
160
|
+
except HydrafetchTimeout:
|
|
161
|
+
raise
|
|
162
|
+
except HydrafetchError as err:
|
|
163
|
+
if err.is_out_of_credits:
|
|
164
|
+
top_up()
|
|
165
|
+
elif err.is_retryable:
|
|
166
|
+
enqueue(url)
|
|
167
|
+
else:
|
|
168
|
+
raise
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
| Status | Meaning | Retried |
|
|
172
|
+
| --- | --- | --- |
|
|
173
|
+
| 400, 422 | invalid request | no |
|
|
174
|
+
| 401, 403 | invalid or missing key | no |
|
|
175
|
+
| 402 | out of credits | no |
|
|
176
|
+
| 429 | rate limited | yes, twice with backoff |
|
|
177
|
+
| 5xx | upstream failure | yes, twice with backoff |
|
|
178
|
+
|
|
179
|
+
A page that loads but answers with an error status raises `ValueError` from the loader instead — that is a bad URL, not a failed request.
|
|
180
|
+
|
|
181
|
+
## Configuration
|
|
182
|
+
|
|
183
|
+
Every class accepts the same connection options, all optional:
|
|
184
|
+
|
|
185
|
+
| option | default | meaning |
|
|
186
|
+
| --- | --- | --- |
|
|
187
|
+
| `api_key` | `HYDRAFETCH_API_KEY` | your API key |
|
|
188
|
+
| `base_url` | `https://api.hydrafetch.com` | API base URL |
|
|
189
|
+
| `timeout` | `120.0` | per-request timeout in seconds |
|
|
190
|
+
| `max_retries` | `2` | retries on 429 and 5xx |
|
|
191
|
+
|
|
192
|
+
## Credits
|
|
193
|
+
|
|
194
|
+
| call | credits |
|
|
195
|
+
| --- | --- |
|
|
196
|
+
| loader, `scrape` mode | 1 |
|
|
197
|
+
| loader, `map` mode | 1 |
|
|
198
|
+
| loader, `crawl` mode | 1 per page |
|
|
199
|
+
| retriever | 1, plus 1 per result with `scrape_content=True` |
|
|
200
|
+
| `hydrafetch_scrape` | 1 |
|
|
201
|
+
| `hydrafetch_search` | 1 |
|
|
202
|
+
| `hydrafetch_extract` | 5 per URL |
|
|
203
|
+
| `hydrafetch_brand` | 5 |
|
|
204
|
+
|
|
205
|
+
Failed requests are not billed. Pricing does not vary with page difficulty, so there is no render, stealth or proxy option to set.
|
|
206
|
+
|
|
207
|
+
## Implementation notes
|
|
208
|
+
|
|
209
|
+
- Prefer `map` then a batch of `scrape` loads over a broad `crawl`. Fetching a whole site and discarding most of it is the most common source of wasted credits.
|
|
210
|
+
- `params` in `crawl` mode is forwarded to the crawl endpoint, so per-page options belong under `scrapeOptions`. At the top level they are ignored.
|
|
211
|
+
- `preferStructure` is off by default. Turn it on when headings, lists and tables matter; leave it off for raw article text.
|
|
212
|
+
- The loader constructs its client eagerly, so a missing API key fails at construction rather than at `load()`.
|
|
213
|
+
- The retriever and tools construct their clients lazily on first use, which keeps them cheap to build and safe to define at import time.
|
|
214
|
+
- Metadata keys are snake_cased on the way out, so the API's `siteName` becomes `site_name`.
|
|
215
|
+
- Scraped content is untrusted input. Do not pass it to a model as instructions, and keep `metadata["source"]` with anything extracted from it.
|
|
216
|
+
|
|
217
|
+
## Development
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
uv sync
|
|
221
|
+
uv run ruff check src tests
|
|
222
|
+
uv run ruff format --check src tests
|
|
223
|
+
uv run pytest -q
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
Unit tests run offline against fake clients. The integration tests are LangChain's own `langchain-tests` standard suite; they are skipped unless `HYDRAFETCH_API_KEY` is set, and they spend real credits.
|
|
227
|
+
|
|
228
|
+
## Links
|
|
229
|
+
|
|
230
|
+
- [Documentation](https://docs.hydrafetch.com)
|
|
231
|
+
- [OpenAPI specification](https://api.hydrafetch.com/openapi.json)
|
|
232
|
+
- [MCP server and editor setup](https://hydrafetch.com/mcp)
|
|
233
|
+
- [Python client](https://github.com/Hydrafetch/python-sdk) — the client this integration wraps
|
|
234
|
+
- [LlamaIndex reader](https://github.com/Hydrafetch/llama-index-readers-hydrafetch)
|
|
235
|
+
- Other clients: [Node](https://github.com/Hydrafetch/node-sdk) · [Go](https://github.com/Hydrafetch/go-sdk) · [Ruby](https://github.com/Hydrafetch/ruby-sdk) · [Rust](https://github.com/Hydrafetch/rust-sdk) · [PHP](https://github.com/Hydrafetch/php-sdk)
|
|
236
|
+
|
|
237
|
+
## License
|
|
238
|
+
|
|
239
|
+
MIT
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "langchain-hydrafetch"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "LangChain integration for Hydrafetch. Load, search and extract clean web data for agents and RAG."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
requires-python = ">=3.10"
|
|
8
|
+
authors = [{ name = "Hydrafetch", email = "team@hydrafetch.com" }]
|
|
9
|
+
keywords = ["langchain", "hydrafetch", "web-scraping", "markdown", "llm", "rag", "agents", "document-loader"]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Development Status :: 4 - Beta",
|
|
12
|
+
"Intended Audience :: Developers",
|
|
13
|
+
"License :: OSI Approved :: MIT License",
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"Topic :: Internet :: WWW/HTTP",
|
|
16
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
17
|
+
"Typing :: Typed",
|
|
18
|
+
]
|
|
19
|
+
dependencies = ["langchain-core>=1.0,<2", "hydrafetch>=0.1.1", "pydantic>=2"]
|
|
20
|
+
|
|
21
|
+
[project.urls]
|
|
22
|
+
Homepage = "https://hydrafetch.com"
|
|
23
|
+
Documentation = "https://docs.hydrafetch.com"
|
|
24
|
+
Source = "https://github.com/Hydrafetch/langchain-hydrafetch"
|
|
25
|
+
Issues = "https://github.com/Hydrafetch/langchain-hydrafetch/issues"
|
|
26
|
+
|
|
27
|
+
[build-system]
|
|
28
|
+
requires = ["hatchling"]
|
|
29
|
+
build-backend = "hatchling.build"
|
|
30
|
+
|
|
31
|
+
[tool.hatch.build.targets.wheel]
|
|
32
|
+
packages = ["src/langchain_hydrafetch"]
|
|
33
|
+
|
|
34
|
+
[dependency-groups]
|
|
35
|
+
dev = ["pytest>=8", "pytest-asyncio>=0.23", "pytest-socket>=0.7", "ruff>=0.6", "langchain-tests>=1.0"]
|
|
36
|
+
|
|
37
|
+
[tool.pytest.ini_options]
|
|
38
|
+
asyncio_mode = "auto"
|
|
39
|
+
testpaths = ["tests"]
|
|
40
|
+
|
|
41
|
+
[tool.ruff]
|
|
42
|
+
line-length = 100
|
|
43
|
+
src = ["src", "tests"]
|
|
44
|
+
|
|
45
|
+
[tool.ruff.lint]
|
|
46
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""LangChain integration for the Hydrafetch web data API."""
|
|
2
|
+
|
|
3
|
+
from .document_loaders import HydrafetchLoader
|
|
4
|
+
from .retrievers import HydrafetchSearchRetriever
|
|
5
|
+
from .tools import (
|
|
6
|
+
HydrafetchBrandTool,
|
|
7
|
+
HydrafetchExtractTool,
|
|
8
|
+
HydrafetchScrapeTool,
|
|
9
|
+
HydrafetchSearchTool,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"HydrafetchBrandTool",
|
|
14
|
+
"HydrafetchExtractTool",
|
|
15
|
+
"HydrafetchLoader",
|
|
16
|
+
"HydrafetchScrapeTool",
|
|
17
|
+
"HydrafetchSearchRetriever",
|
|
18
|
+
"HydrafetchSearchTool",
|
|
19
|
+
]
|
|
20
|
+
__version__ = "0.1.0"
|