langchain-hydrafetch 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,20 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ test:
10
+ runs-on: ubuntu-latest
11
+ strategy:
12
+ matrix:
13
+ python: ['3.10', '3.11', '3.12', '3.13']
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+ - uses: astral-sh/setup-uv@v5
17
+ - run: uv sync --python ${{ matrix.python }}
18
+ - run: uv run ruff check src tests
19
+ - run: uv run ruff format --check src tests
20
+ - run: uv run pytest -q tests/unit_tests
@@ -0,0 +1,27 @@
1
+ name: Publish
2
+
3
+ # Trusted publishing to PyPI: no API token stored anywhere, the id-token proves
4
+ # the release came from this workflow in this repo.
5
+ on:
6
+ push:
7
+ tags: ['v*']
8
+
9
+ jobs:
10
+ publish:
11
+ runs-on: ubuntu-latest
12
+ environment: pypi
13
+ permissions:
14
+ contents: read
15
+ id-token: write
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+ - uses: astral-sh/setup-uv@v5
19
+ - run: uv sync
20
+ - run: uv run pytest -q tests/unit_tests
21
+ - name: Fail if the tag and pyproject disagree
22
+ run: |
23
+ TAG="${GITHUB_REF_NAME#v}"
24
+ PKG=$(uv run python -c "import tomllib;print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
25
+ [ "$TAG" = "$PKG" ] || { echo "tag $TAG != pyproject $PKG"; exit 1; }
26
+ - run: uv build
27
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,9 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ dist/
5
+ build/
6
+ *.egg-info/
7
+ .pytest_cache/
8
+ .ruff_cache/
9
+ .DS_Store
@@ -0,0 +1,9 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Hydrafetch
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
6
+
7
+ The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
8
+
9
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,264 @@
1
+ Metadata-Version: 2.5
2
+ Name: langchain-hydrafetch
3
+ Version: 0.1.0
4
+ Summary: LangChain integration for Hydrafetch. Load, search and extract clean web data for agents and RAG.
5
+ Project-URL: Homepage, https://hydrafetch.com
6
+ Project-URL: Documentation, https://docs.hydrafetch.com
7
+ Project-URL: Source, https://github.com/Hydrafetch/langchain-hydrafetch
8
+ Project-URL: Issues, https://github.com/Hydrafetch/langchain-hydrafetch/issues
9
+ Author-email: Hydrafetch <team@hydrafetch.com>
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: agents,document-loader,hydrafetch,langchain,llm,markdown,rag,web-scraping
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Topic :: Internet :: WWW/HTTP
18
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
19
+ Classifier: Typing :: Typed
20
+ Requires-Python: >=3.10
21
+ Requires-Dist: hydrafetch>=0.1.1
22
+ Requires-Dist: langchain-core<2,>=1.0
23
+ Requires-Dist: pydantic>=2
24
+ Description-Content-Type: text/markdown
25
+
26
+ # langchain-hydrafetch
27
+
28
+ [![PyPI](https://img.shields.io/pypi/v/langchain-hydrafetch)](https://pypi.org/project/langchain-hydrafetch/)
29
+ [![CI](https://github.com/Hydrafetch/langchain-hydrafetch/actions/workflows/ci.yml/badge.svg)](https://github.com/Hydrafetch/langchain-hydrafetch/actions/workflows/ci.yml)
30
+ [![Python](https://img.shields.io/pypi/pyversions/langchain-hydrafetch)](https://pypi.org/project/langchain-hydrafetch/)
31
+
32
+ LangChain integration for the [Hydrafetch](https://hydrafetch.com) web data API.
33
+
34
+ Load pages as documents, search the web as a retriever, and give agents four tools that read, extract and enrich from the live web. Sync and async, fully typed.
35
+
36
+ ## Installation
37
+
38
+ ```bash
39
+ pip install langchain-hydrafetch
40
+ ```
41
+
42
+ Create a key at [app.hydrafetch.com](https://app.hydrafetch.com) and set it:
43
+
44
+ ```bash
45
+ export HYDRAFETCH_API_KEY="hf_..."
46
+ ```
47
+
48
+ Everything below reads that variable when no `api_key=` is passed.
49
+
50
+ ## Quick start
51
+
52
+ ```python
53
+ from langchain_hydrafetch import HydrafetchLoader
54
+
55
+ docs = HydrafetchLoader("https://example.com/article").load()
56
+ print(docs[0].page_content)
57
+ print(docs[0].metadata["title"])
58
+ ```
59
+
60
+ ## Document loader
61
+
62
+ Three modes. `scrape` loads one page, `crawl` walks a site and loads every page it finds, and `map` returns one document per discovered URL without fetching the bodies.
63
+
64
+ ```python
65
+ HydrafetchLoader("https://example.com/article").load()
66
+ HydrafetchLoader("https://example.com", mode="crawl", params={"limit": 50}).load()
67
+ HydrafetchLoader("https://example.com", mode="map").load()
68
+ ```
69
+
70
+ `content_format` chooses what lands in `page_content` — `markdown` by default, or `html`, `rawHtml`, `summary`. `params` is forwarded to the API untouched, so anything the endpoint accepts works here:
71
+
72
+ ```python
73
+ HydrafetchLoader(
74
+ "https://example.com/article",
75
+ content_format="markdown",
76
+ params={"onlyMainContent": True, "preferStructure": True, "blockAds": True},
77
+ ).load()
78
+ ```
79
+
80
+ Documents carry `source` and `status`, plus whatever page metadata was found: `title`, `description`, `language`, `site_name`, `author`, `published_time`, `word_count`, `page_type`, `image`. A redirect adds `final_url`; a crawled page adds `depth`; a mapped URL adds `lastmod` when the sitemap declares one.
81
+
82
+ Use `lazy_load()` to stream a large crawl instead of building the whole list in memory:
83
+
84
+ ```python
85
+ for doc in HydrafetchLoader("https://example.com", mode="crawl").lazy_load():
86
+ index.add(doc)
87
+ ```
88
+
89
+ ### Error pages are refused, not loaded
90
+
91
+ A URL that answers with an error status raises instead of returning a document, because the body of a 404 page is not the page you asked for and a retrieval index should not quietly absorb one.
92
+
93
+ ```python
94
+ HydrafetchLoader("https://example.com/gone").load()
95
+ # ValueError: https://example.com/gone returned HTTP 404. The body of an error
96
+ # page is not the page you asked for; pass raise_for_status=False to load it anyway.
97
+ ```
98
+
99
+ In `crawl` mode a single dead page must not throw away the whole job, so error pages are dropped and the rest of the crawl is kept. Pass `raise_for_status=False` to load error pages in either mode.
100
+
101
+ ## Retriever
102
+
103
+ ```python
104
+ from langchain_hydrafetch import HydrafetchSearchRetriever
105
+
106
+ retriever = HydrafetchSearchRetriever(k=5)
107
+ docs = retriever.invoke("best open source vector databases")
108
+ ```
109
+
110
+ Each document holds the result snippet, with `source`, `title` and `rank` in metadata. Pass `scrape_content=True` to fetch and return the full page body for every result instead:
111
+
112
+ ```python
113
+ HydrafetchSearchRetriever(k=3, scrape_content=True).invoke("...")
114
+ ```
115
+
116
+ That costs one extra credit per result. `k` works as a constructor argument or per call, and `search_params` is forwarded to the search endpoint:
117
+
118
+ ```python
119
+ retriever.invoke("...", k=2)
120
+ HydrafetchSearchRetriever(search_params={"country": "us"})
121
+ ```
122
+
123
+ ## Agent tools
124
+
125
+ ```python
126
+ from langchain_hydrafetch import (
127
+ HydrafetchBrandTool,
128
+ HydrafetchExtractTool,
129
+ HydrafetchScrapeTool,
130
+ HydrafetchSearchTool,
131
+ )
132
+
133
+ tools = [
134
+ HydrafetchSearchTool(),
135
+ HydrafetchScrapeTool(),
136
+ HydrafetchExtractTool(),
137
+ HydrafetchBrandTool(),
138
+ ]
139
+ ```
140
+
141
+ | tool | argument | returns |
142
+ | --- | --- | --- |
143
+ | `hydrafetch_search` | `query`, `limit` | JSON with `query` and `results` of title, url, snippet |
144
+ | `hydrafetch_scrape` | `url` | the page as markdown |
145
+ | `hydrafetch_extract` | `urls`, `json_schema` or `prompt` | JSON with one entry per URL |
146
+ | `hydrafetch_brand` | `domain` | JSON brand record: name, description, tagline, logo assets, colours, fonts, socials |
147
+
148
+ Search finds pages, scrape reads a page you already have. Giving an agent both is the usual setup.
149
+
150
+ Structured extraction takes either a schema or a plain-language description:
151
+
152
+ ```python
153
+ HydrafetchExtractTool().invoke({
154
+ "urls": ["https://example.com/pricing"],
155
+ "json_schema": {
156
+ "type": "object",
157
+ "properties": {"plans": {"type": "array", "items": {"type": "string"}}},
158
+ },
159
+ })
160
+
161
+ HydrafetchExtractTool().invoke({
162
+ "urls": ["https://example.com/about"],
163
+ "prompt": "the founding year and the headquarters city",
164
+ })
165
+ ```
166
+
167
+ ## Async
168
+
169
+ The retriever and every tool support `ainvoke`, and the loader supports `alazy_load` and `aload`:
170
+
171
+ ```python
172
+ docs = await HydrafetchSearchRetriever().ainvoke("...")
173
+ text = await HydrafetchScrapeTool().ainvoke({"url": "https://example.com"})
174
+ ```
175
+
176
+ ## Error handling
177
+
178
+ Failures raise `HydrafetchError` from the underlying client, carrying the API's error code, HTTP status and request id.
179
+
180
+ ```python
181
+ from hydrafetch import HydrafetchError, HydrafetchTimeout
182
+
183
+ try:
184
+ docs = HydrafetchLoader(url).load()
185
+ except HydrafetchTimeout:
186
+ raise
187
+ except HydrafetchError as err:
188
+ if err.is_out_of_credits:
189
+ top_up()
190
+ elif err.is_retryable:
191
+ enqueue(url)
192
+ else:
193
+ raise
194
+ ```
195
+
196
+ | Status | Meaning | Retried |
197
+ | --- | --- | --- |
198
+ | 400, 422 | invalid request | no |
199
+ | 401, 403 | invalid or missing key | no |
200
+ | 402 | out of credits | no |
201
+ | 429 | rate limited | yes, twice with backoff |
202
+ | 5xx | upstream failure | yes, twice with backoff |
203
+
204
+ A page that loads but answers with an error status raises `ValueError` from the loader instead — that is a bad URL, not a failed request.
205
+
206
+ ## Configuration
207
+
208
+ Every class accepts the same connection options, all optional:
209
+
210
+ | option | default | meaning |
211
+ | --- | --- | --- |
212
+ | `api_key` | `HYDRAFETCH_API_KEY` | your API key |
213
+ | `base_url` | `https://api.hydrafetch.com` | API base URL |
214
+ | `timeout` | `120.0` | per-request timeout in seconds |
215
+ | `max_retries` | `2` | retries on 429 and 5xx |
216
+
217
+ ## Credits
218
+
219
+ | call | credits |
220
+ | --- | --- |
221
+ | loader, `scrape` mode | 1 |
222
+ | loader, `map` mode | 1 |
223
+ | loader, `crawl` mode | 1 per page |
224
+ | retriever | 1, plus 1 per result with `scrape_content=True` |
225
+ | `hydrafetch_scrape` | 1 |
226
+ | `hydrafetch_search` | 1 |
227
+ | `hydrafetch_extract` | 5 per URL |
228
+ | `hydrafetch_brand` | 5 |
229
+
230
+ Failed requests are not billed. Pricing does not vary with page difficulty, so there is no render, stealth or proxy option to set.
231
+
232
+ ## Implementation notes
233
+
234
+ - Prefer `map` then a batch of `scrape` loads over a broad `crawl`. Fetching a whole site and discarding most of it is the most common source of wasted credits.
235
+ - `params` in `crawl` mode is forwarded to the crawl endpoint, so per-page options belong under `scrapeOptions`. At the top level they are ignored.
236
+ - `preferStructure` is off by default. Turn it on when headings, lists and tables matter; leave it off for raw article text.
237
+ - The loader constructs its client eagerly, so a missing API key fails at construction rather than at `load()`.
238
+ - The retriever and tools construct their clients lazily on first use, which keeps them cheap to build and safe to define at import time.
239
+ - Metadata keys are snake_cased on the way out, so the API's `siteName` becomes `site_name`.
240
+ - Scraped content is untrusted input. Do not pass it to a model as instructions, and keep `metadata["source"]` with anything extracted from it.
241
+
242
+ ## Development
243
+
244
+ ```bash
245
+ uv sync
246
+ uv run ruff check src tests
247
+ uv run ruff format --check src tests
248
+ uv run pytest -q
249
+ ```
250
+
251
+ Unit tests run offline against fake clients. The integration tests are LangChain's own `langchain-tests` standard suite; they are skipped unless `HYDRAFETCH_API_KEY` is set, and they spend real credits.
252
+
253
+ ## Links
254
+
255
+ - [Documentation](https://docs.hydrafetch.com)
256
+ - [OpenAPI specification](https://api.hydrafetch.com/openapi.json)
257
+ - [MCP server and editor setup](https://hydrafetch.com/mcp)
258
+ - [Python client](https://github.com/Hydrafetch/python-sdk) — the client this integration wraps
259
+ - [LlamaIndex reader](https://github.com/Hydrafetch/llama-index-readers-hydrafetch)
260
+ - Other clients: [Node](https://github.com/Hydrafetch/node-sdk) · [Go](https://github.com/Hydrafetch/go-sdk) · [Ruby](https://github.com/Hydrafetch/ruby-sdk) · [Rust](https://github.com/Hydrafetch/rust-sdk) · [PHP](https://github.com/Hydrafetch/php-sdk)
261
+
262
+ ## License
263
+
264
+ MIT
@@ -0,0 +1,239 @@
1
+ # langchain-hydrafetch
2
+
3
+ [![PyPI](https://img.shields.io/pypi/v/langchain-hydrafetch)](https://pypi.org/project/langchain-hydrafetch/)
4
+ [![CI](https://github.com/Hydrafetch/langchain-hydrafetch/actions/workflows/ci.yml/badge.svg)](https://github.com/Hydrafetch/langchain-hydrafetch/actions/workflows/ci.yml)
5
+ [![Python](https://img.shields.io/pypi/pyversions/langchain-hydrafetch)](https://pypi.org/project/langchain-hydrafetch/)
6
+
7
+ LangChain integration for the [Hydrafetch](https://hydrafetch.com) web data API.
8
+
9
+ Load pages as documents, search the web as a retriever, and give agents four tools that read, extract and enrich from the live web. Sync and async, fully typed.
10
+
11
+ ## Installation
12
+
13
+ ```bash
14
+ pip install langchain-hydrafetch
15
+ ```
16
+
17
+ Create a key at [app.hydrafetch.com](https://app.hydrafetch.com) and set it:
18
+
19
+ ```bash
20
+ export HYDRAFETCH_API_KEY="hf_..."
21
+ ```
22
+
23
+ Everything below reads that variable when no `api_key=` is passed.
24
+
25
+ ## Quick start
26
+
27
+ ```python
28
+ from langchain_hydrafetch import HydrafetchLoader
29
+
30
+ docs = HydrafetchLoader("https://example.com/article").load()
31
+ print(docs[0].page_content)
32
+ print(docs[0].metadata["title"])
33
+ ```
34
+
35
+ ## Document loader
36
+
37
+ Three modes. `scrape` loads one page, `crawl` walks a site and loads every page it finds, and `map` returns one document per discovered URL without fetching the bodies.
38
+
39
+ ```python
40
+ HydrafetchLoader("https://example.com/article").load()
41
+ HydrafetchLoader("https://example.com", mode="crawl", params={"limit": 50}).load()
42
+ HydrafetchLoader("https://example.com", mode="map").load()
43
+ ```
44
+
45
+ `content_format` chooses what lands in `page_content` — `markdown` by default, or `html`, `rawHtml`, `summary`. `params` is forwarded to the API untouched, so anything the endpoint accepts works here:
46
+
47
+ ```python
48
+ HydrafetchLoader(
49
+ "https://example.com/article",
50
+ content_format="markdown",
51
+ params={"onlyMainContent": True, "preferStructure": True, "blockAds": True},
52
+ ).load()
53
+ ```
54
+
55
+ Documents carry `source` and `status`, plus whatever page metadata was found: `title`, `description`, `language`, `site_name`, `author`, `published_time`, `word_count`, `page_type`, `image`. A redirect adds `final_url`; a crawled page adds `depth`; a mapped URL adds `lastmod` when the sitemap declares one.
56
+
57
+ Use `lazy_load()` to stream a large crawl instead of building the whole list in memory:
58
+
59
+ ```python
60
+ for doc in HydrafetchLoader("https://example.com", mode="crawl").lazy_load():
61
+ index.add(doc)
62
+ ```
63
+
64
+ ### Error pages are refused, not loaded
65
+
66
+ A URL that answers with an error status raises instead of returning a document, because the body of a 404 page is not the page you asked for and a retrieval index should not quietly absorb one.
67
+
68
+ ```python
69
+ HydrafetchLoader("https://example.com/gone").load()
70
+ # ValueError: https://example.com/gone returned HTTP 404. The body of an error
71
+ # page is not the page you asked for; pass raise_for_status=False to load it anyway.
72
+ ```
73
+
74
+ In `crawl` mode a single dead page must not throw away the whole job, so error pages are dropped and the rest of the crawl is kept. Pass `raise_for_status=False` to load error pages in either mode.
75
+
76
+ ## Retriever
77
+
78
+ ```python
79
+ from langchain_hydrafetch import HydrafetchSearchRetriever
80
+
81
+ retriever = HydrafetchSearchRetriever(k=5)
82
+ docs = retriever.invoke("best open source vector databases")
83
+ ```
84
+
85
+ Each document holds the result snippet, with `source`, `title` and `rank` in metadata. Pass `scrape_content=True` to fetch and return the full page body for every result instead:
86
+
87
+ ```python
88
+ HydrafetchSearchRetriever(k=3, scrape_content=True).invoke("...")
89
+ ```
90
+
91
+ That costs one extra credit per result. `k` works as a constructor argument or per call, and `search_params` is forwarded to the search endpoint:
92
+
93
+ ```python
94
+ retriever.invoke("...", k=2)
95
+ HydrafetchSearchRetriever(search_params={"country": "us"})
96
+ ```
97
+
98
+ ## Agent tools
99
+
100
+ ```python
101
+ from langchain_hydrafetch import (
102
+ HydrafetchBrandTool,
103
+ HydrafetchExtractTool,
104
+ HydrafetchScrapeTool,
105
+ HydrafetchSearchTool,
106
+ )
107
+
108
+ tools = [
109
+ HydrafetchSearchTool(),
110
+ HydrafetchScrapeTool(),
111
+ HydrafetchExtractTool(),
112
+ HydrafetchBrandTool(),
113
+ ]
114
+ ```
115
+
116
+ | tool | argument | returns |
117
+ | --- | --- | --- |
118
+ | `hydrafetch_search` | `query`, `limit` | JSON with `query` and `results` of title, url, snippet |
119
+ | `hydrafetch_scrape` | `url` | the page as markdown |
120
+ | `hydrafetch_extract` | `urls`, `json_schema` or `prompt` | JSON with one entry per URL |
121
+ | `hydrafetch_brand` | `domain` | JSON brand record: name, description, tagline, logo assets, colours, fonts, socials |
122
+
123
+ Search finds pages, scrape reads a page you already have. Giving an agent both is the usual setup.
124
+
125
+ Structured extraction takes either a schema or a plain-language description:
126
+
127
+ ```python
128
+ HydrafetchExtractTool().invoke({
129
+ "urls": ["https://example.com/pricing"],
130
+ "json_schema": {
131
+ "type": "object",
132
+ "properties": {"plans": {"type": "array", "items": {"type": "string"}}},
133
+ },
134
+ })
135
+
136
+ HydrafetchExtractTool().invoke({
137
+ "urls": ["https://example.com/about"],
138
+ "prompt": "the founding year and the headquarters city",
139
+ })
140
+ ```
141
+
142
+ ## Async
143
+
144
+ The retriever and every tool support `ainvoke`, and the loader supports `alazy_load` and `aload`:
145
+
146
+ ```python
147
+ docs = await HydrafetchSearchRetriever().ainvoke("...")
148
+ text = await HydrafetchScrapeTool().ainvoke({"url": "https://example.com"})
149
+ ```
150
+
151
+ ## Error handling
152
+
153
+ Failures raise `HydrafetchError` from the underlying client, carrying the API's error code, HTTP status and request id.
154
+
155
+ ```python
156
+ from hydrafetch import HydrafetchError, HydrafetchTimeout
157
+
158
+ try:
159
+ docs = HydrafetchLoader(url).load()
160
+ except HydrafetchTimeout:
161
+ raise
162
+ except HydrafetchError as err:
163
+ if err.is_out_of_credits:
164
+ top_up()
165
+ elif err.is_retryable:
166
+ enqueue(url)
167
+ else:
168
+ raise
169
+ ```
170
+
171
+ | Status | Meaning | Retried |
172
+ | --- | --- | --- |
173
+ | 400, 422 | invalid request | no |
174
+ | 401, 403 | invalid or missing key | no |
175
+ | 402 | out of credits | no |
176
+ | 429 | rate limited | yes, twice with backoff |
177
+ | 5xx | upstream failure | yes, twice with backoff |
178
+
179
+ A page that loads but answers with an error status raises `ValueError` from the loader instead — that is a bad URL, not a failed request.
180
+
181
+ ## Configuration
182
+
183
+ Every class accepts the same connection options, all optional:
184
+
185
+ | option | default | meaning |
186
+ | --- | --- | --- |
187
+ | `api_key` | `HYDRAFETCH_API_KEY` | your API key |
188
+ | `base_url` | `https://api.hydrafetch.com` | API base URL |
189
+ | `timeout` | `120.0` | per-request timeout in seconds |
190
+ | `max_retries` | `2` | retries on 429 and 5xx |
191
+
192
+ ## Credits
193
+
194
+ | call | credits |
195
+ | --- | --- |
196
+ | loader, `scrape` mode | 1 |
197
+ | loader, `map` mode | 1 |
198
+ | loader, `crawl` mode | 1 per page |
199
+ | retriever | 1, plus 1 per result with `scrape_content=True` |
200
+ | `hydrafetch_scrape` | 1 |
201
+ | `hydrafetch_search` | 1 |
202
+ | `hydrafetch_extract` | 5 per URL |
203
+ | `hydrafetch_brand` | 5 |
204
+
205
+ Failed requests are not billed. Pricing does not vary with page difficulty, so there is no render, stealth or proxy option to set.
206
+
207
+ ## Implementation notes
208
+
209
+ - Prefer `map` then a batch of `scrape` loads over a broad `crawl`. Fetching a whole site and discarding most of it is the most common source of wasted credits.
210
+ - `params` in `crawl` mode is forwarded to the crawl endpoint, so per-page options belong under `scrapeOptions`. At the top level they are ignored.
211
+ - `preferStructure` is off by default. Turn it on when headings, lists and tables matter; leave it off for raw article text.
212
+ - The loader constructs its client eagerly, so a missing API key fails at construction rather than at `load()`.
213
+ - The retriever and tools construct their clients lazily on first use, which keeps them cheap to build and safe to define at import time.
214
+ - Metadata keys are snake_cased on the way out, so the API's `siteName` becomes `site_name`.
215
+ - Scraped content is untrusted input. Do not pass it to a model as instructions, and keep `metadata["source"]` with anything extracted from it.
216
+
217
+ ## Development
218
+
219
+ ```bash
220
+ uv sync
221
+ uv run ruff check src tests
222
+ uv run ruff format --check src tests
223
+ uv run pytest -q
224
+ ```
225
+
226
+ Unit tests run offline against fake clients. The integration tests are LangChain's own `langchain-tests` standard suite; they are skipped unless `HYDRAFETCH_API_KEY` is set, and they spend real credits.
227
+
228
+ ## Links
229
+
230
+ - [Documentation](https://docs.hydrafetch.com)
231
+ - [OpenAPI specification](https://api.hydrafetch.com/openapi.json)
232
+ - [MCP server and editor setup](https://hydrafetch.com/mcp)
233
+ - [Python client](https://github.com/Hydrafetch/python-sdk) — the client this integration wraps
234
+ - [LlamaIndex reader](https://github.com/Hydrafetch/llama-index-readers-hydrafetch)
235
+ - Other clients: [Node](https://github.com/Hydrafetch/node-sdk) · [Go](https://github.com/Hydrafetch/go-sdk) · [Ruby](https://github.com/Hydrafetch/ruby-sdk) · [Rust](https://github.com/Hydrafetch/rust-sdk) · [PHP](https://github.com/Hydrafetch/php-sdk)
236
+
237
+ ## License
238
+
239
+ MIT
@@ -0,0 +1,46 @@
1
+ [project]
2
+ name = "langchain-hydrafetch"
3
+ version = "0.1.0"
4
+ description = "LangChain integration for Hydrafetch. Load, search and extract clean web data for agents and RAG."
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ requires-python = ">=3.10"
8
+ authors = [{ name = "Hydrafetch", email = "team@hydrafetch.com" }]
9
+ keywords = ["langchain", "hydrafetch", "web-scraping", "markdown", "llm", "rag", "agents", "document-loader"]
10
+ classifiers = [
11
+ "Development Status :: 4 - Beta",
12
+ "Intended Audience :: Developers",
13
+ "License :: OSI Approved :: MIT License",
14
+ "Programming Language :: Python :: 3",
15
+ "Topic :: Internet :: WWW/HTTP",
16
+ "Topic :: Software Development :: Libraries :: Python Modules",
17
+ "Typing :: Typed",
18
+ ]
19
+ dependencies = ["langchain-core>=1.0,<2", "hydrafetch>=0.1.1", "pydantic>=2"]
20
+
21
+ [project.urls]
22
+ Homepage = "https://hydrafetch.com"
23
+ Documentation = "https://docs.hydrafetch.com"
24
+ Source = "https://github.com/Hydrafetch/langchain-hydrafetch"
25
+ Issues = "https://github.com/Hydrafetch/langchain-hydrafetch/issues"
26
+
27
+ [build-system]
28
+ requires = ["hatchling"]
29
+ build-backend = "hatchling.build"
30
+
31
+ [tool.hatch.build.targets.wheel]
32
+ packages = ["src/langchain_hydrafetch"]
33
+
34
+ [dependency-groups]
35
+ dev = ["pytest>=8", "pytest-asyncio>=0.23", "pytest-socket>=0.7", "ruff>=0.6", "langchain-tests>=1.0"]
36
+
37
+ [tool.pytest.ini_options]
38
+ asyncio_mode = "auto"
39
+ testpaths = ["tests"]
40
+
41
+ [tool.ruff]
42
+ line-length = 100
43
+ src = ["src", "tests"]
44
+
45
+ [tool.ruff.lint]
46
+ select = ["E", "F", "I", "UP", "B"]
@@ -0,0 +1,20 @@
1
+ """LangChain integration for the Hydrafetch web data API."""
2
+
3
+ from .document_loaders import HydrafetchLoader
4
+ from .retrievers import HydrafetchSearchRetriever
5
+ from .tools import (
6
+ HydrafetchBrandTool,
7
+ HydrafetchExtractTool,
8
+ HydrafetchScrapeTool,
9
+ HydrafetchSearchTool,
10
+ )
11
+
12
+ __all__ = [
13
+ "HydrafetchBrandTool",
14
+ "HydrafetchExtractTool",
15
+ "HydrafetchLoader",
16
+ "HydrafetchScrapeTool",
17
+ "HydrafetchSearchRetriever",
18
+ "HydrafetchSearchTool",
19
+ ]
20
+ __version__ = "0.1.0"