langchain-clawfetch 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ *.egg-info/
4
+ dist/
5
+ build/
6
+ .pytest_cache/
7
+ .venv/
@@ -0,0 +1,6 @@
1
+ # Changelog
2
+
3
+ ## [0.1.0] - 2026-10-18
4
+
5
+ ### Added
6
+ - Initial release: `ClawFetchFetch`, `ClawFetchRender`, `ClawFetchExtract`, `ClawFetchResearch`, `ClawFetchParse`, `ClawFetchDomainsCheck`, `ClawFetchDomainsSuggest` tools (sync and async) and `ClawFetchToolkit`.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 ClawFetch
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,93 @@
1
+ Metadata-Version: 2.5
2
+ Name: langchain-clawfetch
3
+ Version: 0.1.0
4
+ Summary: LangChain tools for ClawFetch — pay-per-call web intelligence for AI agents (x402, USDC on Base)
5
+ Project-URL: Homepage, https://api.clawfetch.ai
6
+ License-Expression: MIT
7
+ License-File: LICENSE
8
+ Keywords: ai-agents,clawfetch,langchain,tools,web-scraping,x402
9
+ Classifier: Development Status :: 3 - Alpha
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
13
+ Requires-Python: >=3.10
14
+ Requires-Dist: clawfetch<0.3,>=0.2.2
15
+ Requires-Dist: langchain-core<2,>=0.3
16
+ Provides-Extra: test
17
+ Requires-Dist: pytest-asyncio>=0.24; extra == 'test'
18
+ Requires-Dist: pytest>=8; extra == 'test'
19
+ Description-Content-Type: text/markdown
20
+
21
+ # langchain-clawfetch
22
+
23
+ LangChain tools for [ClawFetch](https://api.clawfetch.ai), a pay-per-call web intelligence API for AI agents. Each call is paid with an x402 micropayment in USDC on Base.
24
+
25
+ ## Install
26
+
27
+ ```bash
28
+ pip install langchain-clawfetch
29
+ ```
30
+
31
+ ## Wallet
32
+
33
+ You need a Base wallet private key that holds USDC. There is no API key and no signup. Payments are gasless EIP-3009 transfers signed locally by the [`clawfetch`](https://pypi.org/project/clawfetch/) SDK, so the wallet does not need ETH. Use a dedicated wallet with a small balance.
34
+
35
+ ```bash
36
+ export CLAWFETCH_PRIVATE_KEY=0x...
37
+ ```
38
+
39
+ ## Quickstart
40
+
41
+ ```python
42
+ from langchain_clawfetch import ClawFetchToolkit
43
+
44
+ tools = ClawFetchToolkit().get_tools() # reads CLAWFETCH_PRIVATE_KEY
45
+ # or: ClawFetchToolkit(private_key="0x...", max_chars=8000)
46
+
47
+ fetch = tools[0]
48
+ print(fetch.invoke({"url": "https://example.com"}))
49
+ ```
50
+
51
+ To use the tools with an agent:
52
+
53
+ ```python
54
+ from langchain.agents import create_agent
55
+
56
+ agent = create_agent(llm, tools=tools) # llm: any LangChain chat model
57
+ agent.invoke({"messages": [{"role": "user", "content": "What's new on news.ycombinator.com?"}]})
58
+ ```
59
+
60
+ You can also create a single tool, or pass an existing client:
61
+
62
+ ```python
63
+ from clawfetch import ClawFetch, AsyncClawFetch
64
+ from langchain_clawfetch import ClawFetchResearch
65
+
66
+ research = ClawFetchResearch(client=ClawFetch(private_key="0x..."), async_client=AsyncClawFetch(private_key="0x..."))
67
+ ```
68
+
69
+ Every tool supports `invoke` and `ainvoke`. Async calls use `AsyncClawFetch`.
70
+
71
+ ## Tools
72
+
73
+ | Tool | Name | Price per call | What it does |
74
+ |---|---|---|---|
75
+ | `ClawFetchFetch` | `clawfetch_fetch` | $0.001 | URL to readable text (no JavaScript) |
76
+ | `ClawFetchRender` | `clawfetch_render` | $0.005 | JavaScript-rendered page to text |
77
+ | `ClawFetchExtract` | `clawfetch_extract` | $0.008 | Typed JSON from 17 supported sites |
78
+ | `ClawFetchResearch` | `clawfetch_research` | $0.02 | Web search plus source texts for citations |
79
+ | `ClawFetchParse` | `clawfetch_parse` | $0.005 | Office document or PDF at a URL to Markdown (no OCR) |
80
+ | `ClawFetchDomainsCheck` | `clawfetch_domains_check` | $0.003 | Domain availability |
81
+ | `ClawFetchDomainsSuggest` | `clawfetch_domains_suggest` | $0.003 | Generate and check domain names from keywords |
82
+
83
+ Supported `clawfetch_extract` sites: twitter, coingecko, github, news, npm, pypi, crates, hackernews, youtube, wikipedia, producthunt, imdb, amazon, crunchbase, sec-edgar, zillow, redfin.
84
+
85
+ ## Output and errors
86
+
87
+ Tools return a string. Fetch and render return the page title, URL and text. Parse returns Markdown. The other tools return the API response as JSON. Output is cut at `max_chars` (default 10,000).
88
+
89
+ SDK errors (`PaymentError`, `RateLimitError`, `ApiError`, `NetworkError`) are raised as `ToolException`. To hand the message to the model instead of raising, pass `handle_tool_error=True` when you create a tool.
90
+
91
+ ## License
92
+
93
+ MIT
@@ -0,0 +1,73 @@
1
+ # langchain-clawfetch
2
+
3
+ LangChain tools for [ClawFetch](https://api.clawfetch.ai), a pay-per-call web intelligence API for AI agents. Each call is paid with an x402 micropayment in USDC on Base.
4
+
5
+ ## Install
6
+
7
+ ```bash
8
+ pip install langchain-clawfetch
9
+ ```
10
+
11
+ ## Wallet
12
+
13
+ You need a Base wallet private key that holds USDC. There is no API key and no signup. Payments are gasless EIP-3009 transfers signed locally by the [`clawfetch`](https://pypi.org/project/clawfetch/) SDK, so the wallet does not need ETH. Use a dedicated wallet with a small balance.
14
+
15
+ ```bash
16
+ export CLAWFETCH_PRIVATE_KEY=0x...
17
+ ```
18
+
19
+ ## Quickstart
20
+
21
+ ```python
22
+ from langchain_clawfetch import ClawFetchToolkit
23
+
24
+ tools = ClawFetchToolkit().get_tools() # reads CLAWFETCH_PRIVATE_KEY
25
+ # or: ClawFetchToolkit(private_key="0x...", max_chars=8000)
26
+
27
+ fetch = tools[0]
28
+ print(fetch.invoke({"url": "https://example.com"}))
29
+ ```
30
+
31
+ To use the tools with an agent:
32
+
33
+ ```python
34
+ from langchain.agents import create_agent
35
+
36
+ agent = create_agent(llm, tools=tools) # llm: any LangChain chat model
37
+ agent.invoke({"messages": [{"role": "user", "content": "What's new on news.ycombinator.com?"}]})
38
+ ```
39
+
40
+ You can also create a single tool, or pass an existing client:
41
+
42
+ ```python
43
+ from clawfetch import ClawFetch, AsyncClawFetch
44
+ from langchain_clawfetch import ClawFetchResearch
45
+
46
+ research = ClawFetchResearch(client=ClawFetch(private_key="0x..."), async_client=AsyncClawFetch(private_key="0x..."))
47
+ ```
48
+
49
+ Every tool supports `invoke` and `ainvoke`. Async calls use `AsyncClawFetch`.
50
+
51
+ ## Tools
52
+
53
+ | Tool | Name | Price per call | What it does |
54
+ |---|---|---|---|
55
+ | `ClawFetchFetch` | `clawfetch_fetch` | $0.001 | URL to readable text (no JavaScript) |
56
+ | `ClawFetchRender` | `clawfetch_render` | $0.005 | JavaScript-rendered page to text |
57
+ | `ClawFetchExtract` | `clawfetch_extract` | $0.008 | Typed JSON from 17 supported sites |
58
+ | `ClawFetchResearch` | `clawfetch_research` | $0.02 | Web search plus source texts for citations |
59
+ | `ClawFetchParse` | `clawfetch_parse` | $0.005 | Office document or PDF at a URL to Markdown (no OCR) |
60
+ | `ClawFetchDomainsCheck` | `clawfetch_domains_check` | $0.003 | Domain availability |
61
+ | `ClawFetchDomainsSuggest` | `clawfetch_domains_suggest` | $0.003 | Generate and check domain names from keywords |
62
+
63
+ Supported `clawfetch_extract` sites: twitter, coingecko, github, news, npm, pypi, crates, hackernews, youtube, wikipedia, producthunt, imdb, amazon, crunchbase, sec-edgar, zillow, redfin.
64
+
65
+ ## Output and errors
66
+
67
+ Tools return a string. Fetch and render return the page title, URL and text. Parse returns Markdown. The other tools return the API response as JSON. Output is cut at `max_chars` (default 10,000).
68
+
69
+ SDK errors (`PaymentError`, `RateLimitError`, `ApiError`, `NetworkError`) are raised as `ToolException`. To hand the message to the model instead of raising, pass `handle_tool_error=True` when you create a tool.
70
+
71
+ ## License
72
+
73
+ MIT
@@ -0,0 +1,24 @@
1
+ """LangChain integration for ClawFetch — pay-per-call web intelligence for AI agents."""
2
+
3
+ from langchain_clawfetch.tools import (
4
+ ClawFetchDomainsCheck,
5
+ ClawFetchDomainsSuggest,
6
+ ClawFetchExtract,
7
+ ClawFetchFetch,
8
+ ClawFetchParse,
9
+ ClawFetchRender,
10
+ ClawFetchResearch,
11
+ ClawFetchToolkit,
12
+ )
13
+
14
+ __all__ = [
15
+ "ClawFetchFetch",
16
+ "ClawFetchRender",
17
+ "ClawFetchExtract",
18
+ "ClawFetchResearch",
19
+ "ClawFetchParse",
20
+ "ClawFetchDomainsCheck",
21
+ "ClawFetchDomainsSuggest",
22
+ "ClawFetchToolkit",
23
+ ]
24
+ __version__ = "0.1.0"
File without changes
@@ -0,0 +1,272 @@
1
+ """LangChain tools for the ClawFetch web intelligence API (x402, USDC on Base)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ from typing import Any, List, Optional, Tuple, Type
8
+
9
+ from clawfetch import AsyncClawFetch, ClawFetch, ClawFetchError
10
+ from langchain_core.tools import BaseTool, BaseToolkit, ToolException
11
+ from pydantic import BaseModel, ConfigDict, Field, SecretStr
12
+
13
+ DEFAULT_MAX_CHARS = 10_000
14
+
15
+ EXTRACT_TYPES = (
16
+ "twitter, coingecko, github, news, npm, pypi, crates, hackernews, youtube, "
17
+ "wikipedia, producthunt, imdb, amazon, crunchbase, sec-edgar, zillow, redfin"
18
+ )
19
+
20
+
21
+ def _make_clients(private_key: Optional[SecretStr]) -> Tuple[ClawFetch, AsyncClawFetch]:
22
+ key = private_key.get_secret_value() if private_key else os.environ.get("CLAWFETCH_PRIVATE_KEY")
23
+ if not key:
24
+ raise ValueError("Pass private_key=..., set CLAWFETCH_PRIVATE_KEY, or pass client=ClawFetch(...).")
25
+ return ClawFetch(private_key=key), AsyncClawFetch(private_key=key)
26
+
27
+
28
+ def _truncate(text: str, max_chars: int) -> str:
29
+ if len(text) <= max_chars:
30
+ return text
31
+ return text[:max_chars] + f"\n...[truncated {len(text) - max_chars} chars]"
32
+
33
+
34
+ def _json(result: Any) -> str:
35
+ return json.dumps(result, ensure_ascii=False)
36
+
37
+
38
+ def _page(result: dict) -> str:
39
+ """Format a /fetch or /render result as markdown-ish text."""
40
+ title = result.get("title") or ""
41
+ url = result.get("url") or ""
42
+ text = result.get("text") or ""
43
+ return f"# {title}\nURL: {url}\n\n{text}"
44
+
45
+
46
+ # ─── Input schemas ────────────────────────────────────────────────
47
+
48
+
49
+ class FetchInput(BaseModel):
50
+ url: str = Field(description="Absolute http(s) URL of the page to fetch.")
51
+
52
+
53
+ class RenderInput(BaseModel):
54
+ url: str = Field(description="Absolute http(s) URL of the JavaScript-heavy page.")
55
+ wait_for: Optional[str] = Field(
56
+ default=None, description="Optional CSS selector to wait for before reading the page."
57
+ )
58
+
59
+
60
+ class ExtractInput(BaseModel):
61
+ url: str = Field(description="URL on a supported site (e.g. a GitHub repo or CoinGecko coin page).")
62
+ type: Optional[str] = Field(
63
+ default=None,
64
+ description=f"Optional extractor name; auto-detected from the URL when omitted. One of: {EXTRACT_TYPES}.",
65
+ )
66
+
67
+
68
+ class ResearchInput(BaseModel):
69
+ query: str = Field(description="Research question or topic.")
70
+ max_results: Optional[int] = Field(
71
+ default=None, ge=1, le=10, description="Number of sources to read (1-10, default 5)."
72
+ )
73
+
74
+
75
+ class ParseInput(BaseModel):
76
+ url: str = Field(description="Absolute http(s) URL of an office document or PDF.")
77
+ format: Optional[str] = Field(
78
+ default=None,
79
+ description="Optional format override (doc, docx, odt, pdf, ppt, pptx, rtf, epub, xlsx, ods, odp, csv).",
80
+ )
81
+
82
+
83
+ class DomainsCheckInput(BaseModel):
84
+ domains: List[str] = Field(description="Fully qualified domain names to check, e.g. ['acme.ai', 'acme.com'] (max 20).")
85
+
86
+
87
+ class DomainsSuggestInput(BaseModel):
88
+ keywords: List[str] = Field(description="Keywords to combine into domain names, e.g. ['agent', 'grid'].")
89
+ tlds: Optional[List[str]] = Field(
90
+ default=None, description="TLDs to try without dots, e.g. ['com', 'ai']. Default: com, ai, io."
91
+ )
92
+ max_check: Optional[int] = Field(
93
+ default=None, ge=1, le=50, description="How many generated names to check for availability (1-50, default 30)."
94
+ )
95
+
96
+
97
+ # ─── Tools ────────────────────────────────────────────────────────
98
+
99
+
100
+ class _ClawFetchTool(BaseTool):
101
+ """Shared client wiring, error mapping and output formatting."""
102
+
103
+ model_config = ConfigDict(arbitrary_types_allowed=True)
104
+
105
+ private_key: Optional[SecretStr] = Field(default=None, exclude=True)
106
+ client: Optional[ClawFetch] = Field(default=None, exclude=True)
107
+ async_client: Optional[AsyncClawFetch] = Field(default=None, exclude=True)
108
+ max_chars: int = DEFAULT_MAX_CHARS
109
+
110
+ def model_post_init(self, __context: Any) -> None:
111
+ if self.client is None and self.async_client is None:
112
+ self.client, self.async_client = _make_clients(self.private_key)
113
+
114
+ def _invoke(self, client: Any, **kwargs: Any) -> Any:
115
+ raise NotImplementedError
116
+
117
+ def _format(self, result: Any) -> str:
118
+ return _json(result)
119
+
120
+ def _run(self, run_manager: Any = None, **kwargs: Any) -> str:
121
+ if self.client is None:
122
+ raise ToolException(f"{self.name}: no sync ClawFetch client configured; use the async API.")
123
+ try:
124
+ result = self._invoke(self.client, **kwargs)
125
+ except ClawFetchError as exc:
126
+ raise ToolException(f"{self.name} failed ({type(exc).__name__}): {exc}") from exc
127
+ return _truncate(self._format(result), self.max_chars)
128
+
129
+ async def _arun(self, run_manager: Any = None, **kwargs: Any) -> str:
130
+ if self.async_client is None:
131
+ return await super()._arun(**kwargs)
132
+ try:
133
+ result = await self._invoke(self.async_client, **kwargs)
134
+ except ClawFetchError as exc:
135
+ raise ToolException(f"{self.name} failed ({type(exc).__name__}): {exc}") from exc
136
+ return _truncate(self._format(result), self.max_chars)
137
+
138
+
139
+ class ClawFetchFetch(_ClawFetchTool):
140
+ name: str = "clawfetch_fetch"
141
+ description: str = (
142
+ "Fetch a web page and return its readable text with title and URL. "
143
+ "Fast, no JavaScript execution; use clawfetch_render for JS-heavy pages. "
144
+ "Costs $0.001 USDC per call."
145
+ )
146
+ args_schema: Type[BaseModel] = FetchInput
147
+
148
+ def _invoke(self, client: Any, url: str) -> Any:
149
+ return client.fetch(url, max_chars=self.max_chars)
150
+
151
+ def _format(self, result: Any) -> str:
152
+ return _page(result)
153
+
154
+
155
+ class ClawFetchRender(_ClawFetchTool):
156
+ name: str = "clawfetch_render"
157
+ description: str = (
158
+ "Render a JavaScript-heavy page in a headless browser and return its visible text. "
159
+ "Use when clawfetch_fetch returns empty or incomplete content. Costs $0.005 USDC per call."
160
+ )
161
+ args_schema: Type[BaseModel] = RenderInput
162
+
163
+ def _invoke(self, client: Any, url: str, wait_for: Optional[str] = None) -> Any:
164
+ return client.render(url, max_chars=self.max_chars, wait_for=wait_for)
165
+
166
+ def _format(self, result: Any) -> str:
167
+ return _page(result)
168
+
169
+
170
+ class ClawFetchExtract(_ClawFetchTool):
171
+ name: str = "clawfetch_extract"
172
+ description: str = (
173
+ "Extract typed structured data (JSON) from a page on a supported site: "
174
+ f"{EXTRACT_TYPES}. Costs $0.008 USDC per call."
175
+ )
176
+ args_schema: Type[BaseModel] = ExtractInput
177
+
178
+ def _invoke(self, client: Any, url: str, type: Optional[str] = None) -> Any:
179
+ return client.extract(url, type=type)
180
+
181
+
182
+ class ClawFetchResearch(_ClawFetchTool):
183
+ name: str = "clawfetch_research"
184
+ description: str = (
185
+ "Search the web for a query and return the text of the top sources with their URLs "
186
+ "(for citations). Costs $0.02 USDC per call."
187
+ )
188
+ args_schema: Type[BaseModel] = ResearchInput
189
+
190
+ def _invoke(self, client: Any, query: str, max_results: Optional[int] = None) -> Any:
191
+ return client.research(query, max_results=max_results)
192
+
193
+
194
+ class ClawFetchParse(_ClawFetchTool):
195
+ name: str = "clawfetch_parse"
196
+ description: str = (
197
+ "Convert an office document or PDF at a URL (docx, pptx, xlsx, pdf, odt, rtf, epub, csv, ...) "
198
+ "to Markdown. No OCR: scanned PDFs fail. Costs $0.005 USDC per call."
199
+ )
200
+ args_schema: Type[BaseModel] = ParseInput
201
+
202
+ def _invoke(self, client: Any, url: str, format: Optional[str] = None) -> Any:
203
+ return client.parse(url, format=format)
204
+
205
+ def _format(self, result: Any) -> str:
206
+ return result.get("markdown") or ""
207
+
208
+
209
+ class ClawFetchDomainsCheck(_ClawFetchTool):
210
+ name: str = "clawfetch_domains_check"
211
+ description: str = (
212
+ "Check whether specific domain names are available to register. Returns JSON with a "
213
+ "result per domain (available true/false, or null with an error). Costs $0.003 USDC per call."
214
+ )
215
+ args_schema: Type[BaseModel] = DomainsCheckInput
216
+
217
+ def _invoke(self, client: Any, domains: List[str]) -> Any:
218
+ return client.domains_check(domains)
219
+
220
+
221
+ class ClawFetchDomainsSuggest(_ClawFetchTool):
222
+ name: str = "clawfetch_domains_suggest"
223
+ description: str = (
224
+ "Generate domain names from keywords and check which are available. Returns JSON with the "
225
+ "list of available domains and every checked result. Costs $0.003 USDC per call."
226
+ )
227
+ args_schema: Type[BaseModel] = DomainsSuggestInput
228
+
229
+ def _invoke(
230
+ self,
231
+ client: Any,
232
+ keywords: List[str],
233
+ tlds: Optional[List[str]] = None,
234
+ max_check: Optional[int] = None,
235
+ ) -> Any:
236
+ return client.domains_suggest(keywords, tlds=tlds, max_check=max_check)
237
+
238
+
239
+ TOOL_CLASSES = (
240
+ ClawFetchFetch,
241
+ ClawFetchRender,
242
+ ClawFetchExtract,
243
+ ClawFetchResearch,
244
+ ClawFetchParse,
245
+ ClawFetchDomainsCheck,
246
+ ClawFetchDomainsSuggest,
247
+ )
248
+
249
+
250
+ class ClawFetchToolkit(BaseToolkit):
251
+ """All ClawFetch tools sharing one sync and one async client.
252
+
253
+ Build from ``private_key=`` (or the ``CLAWFETCH_PRIVATE_KEY`` env var), or
254
+ pass existing ``client=`` / ``async_client=`` instances.
255
+ """
256
+
257
+ model_config = ConfigDict(arbitrary_types_allowed=True)
258
+
259
+ private_key: Optional[SecretStr] = Field(default=None, exclude=True)
260
+ client: Optional[ClawFetch] = Field(default=None, exclude=True)
261
+ async_client: Optional[AsyncClawFetch] = Field(default=None, exclude=True)
262
+ max_chars: int = DEFAULT_MAX_CHARS
263
+
264
+ def model_post_init(self, __context: Any) -> None:
265
+ if self.client is None and self.async_client is None:
266
+ self.client, self.async_client = _make_clients(self.private_key)
267
+
268
+ def get_tools(self) -> List[BaseTool]:
269
+ return [
270
+ cls(client=self.client, async_client=self.async_client, max_chars=self.max_chars)
271
+ for cls in TOOL_CLASSES
272
+ ]
@@ -0,0 +1,39 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "langchain-clawfetch"
7
+ version = "0.1.0"
8
+ description = "LangChain tools for ClawFetch — pay-per-call web intelligence for AI agents (x402, USDC on Base)"
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ requires-python = ">=3.10"
13
+ keywords = ["langchain", "clawfetch", "x402", "web-scraping", "ai-agents", "tools"]
14
+ classifiers = [
15
+ "Development Status :: 3 - Alpha",
16
+ "Intended Audience :: Developers",
17
+ "Programming Language :: Python :: 3",
18
+ "Topic :: Software Development :: Libraries :: Python Modules",
19
+ ]
20
+ dependencies = [
21
+ "clawfetch>=0.2.2,<0.3",
22
+ "langchain-core>=0.3,<2",
23
+ ]
24
+
25
+ [project.optional-dependencies]
26
+ test = ["pytest>=8", "pytest-asyncio>=0.24"]
27
+
28
+ [project.urls]
29
+ Homepage = "https://api.clawfetch.ai"
30
+
31
+ [tool.hatch.build.targets.wheel]
32
+ packages = ["langchain_clawfetch"]
33
+
34
+ [tool.hatch.build.targets.sdist]
35
+ include = ["langchain_clawfetch/", "tests/", "README.md", "CHANGELOG.md", "LICENSE"]
36
+
37
+ [tool.pytest.ini_options]
38
+ testpaths = ["tests"]
39
+ asyncio_mode = "auto"
File without changes
@@ -0,0 +1,129 @@
1
+ """Unit tests — the ClawFetch client is mocked: no network, no payments."""
2
+
3
+ import json
4
+ from unittest.mock import AsyncMock, MagicMock
5
+
6
+ import pytest
7
+ from clawfetch import AsyncClawFetch, ClawFetch, PaymentError
8
+ from langchain_core.tools import ToolException
9
+
10
+ from langchain_clawfetch import (
11
+ ClawFetchDomainsCheck,
12
+ ClawFetchDomainsSuggest,
13
+ ClawFetchExtract,
14
+ ClawFetchFetch,
15
+ ClawFetchParse,
16
+ ClawFetchRender,
17
+ ClawFetchResearch,
18
+ ClawFetchToolkit,
19
+ )
20
+
21
+ DUMMY_KEY = "0x0000000000000000000000000000000000000000000000000000000000000001"
22
+ PAGE = {"url": "https://a.b", "title": "T", "text": "hello"}
23
+
24
+ # (tool class, tool input, sdk method, expected positional args, expected kwargs, sdk result, expected output)
25
+ CASES = [
26
+ (ClawFetchFetch, {"url": "https://a.b"}, "fetch", ("https://a.b",), {"max_chars": 10_000},
27
+ PAGE, "# T\nURL: https://a.b\n\nhello"),
28
+ (ClawFetchRender, {"url": "https://a.b", "wait_for": "#app"}, "render", ("https://a.b",),
29
+ {"max_chars": 10_000, "wait_for": "#app"}, PAGE, "# T\nURL: https://a.b\n\nhello"),
30
+ (ClawFetchExtract, {"url": "https://github.com/a/b"}, "extract", ("https://github.com/a/b",),
31
+ {"type": None}, {"type": "github", "stars": 1}, json.dumps({"type": "github", "stars": 1})),
32
+ (ClawFetchResearch, {"query": "x402", "max_results": 3}, "research", ("x402",), {"max_results": 3},
33
+ {"query": "x402", "sources": 0, "results": []}, json.dumps({"query": "x402", "sources": 0, "results": []})),
34
+ (ClawFetchParse, {"url": "https://a.b/x.docx"}, "parse", ("https://a.b/x.docx",), {"format": None},
35
+ {"markdown": "# Doc", "format": "docx", "chars": 5}, "# Doc"),
36
+ (ClawFetchDomainsCheck, {"domains": ["a.ai"]}, "domains_check", (["a.ai"],), {},
37
+ {"checked": 1, "available": 1, "taken": 0, "results": [{"domain": "a.ai", "available": True}]},
38
+ json.dumps({"checked": 1, "available": 1, "taken": 0, "results": [{"domain": "a.ai", "available": True}]})),
39
+ (ClawFetchDomainsSuggest, {"keywords": ["agent"], "tlds": ["ai"], "max_check": 5}, "domains_suggest",
40
+ (["agent"],), {"tlds": ["ai"], "max_check": 5},
41
+ {"keywords": ["agent"], "available": ["agent.ai"]}, json.dumps({"keywords": ["agent"], "available": ["agent.ai"]})),
42
+ ]
43
+
44
+
45
+ def _clients():
46
+ return MagicMock(spec=ClawFetch), MagicMock(spec=AsyncClawFetch)
47
+
48
+
49
+ @pytest.mark.parametrize("cls,tool_input,method,args,kwargs,result,expected", CASES)
50
+ def test_sync_tool_calls_sdk_and_formats(cls, tool_input, method, args, kwargs, result, expected):
51
+ client, aclient = _clients()
52
+ getattr(client, method).return_value = result
53
+ tool = cls(client=client, async_client=aclient)
54
+ assert tool.invoke(tool_input) == expected
55
+ getattr(client, method).assert_called_once_with(*args, **kwargs)
56
+
57
+
58
+ @pytest.mark.parametrize("cls,tool_input,method,args,kwargs,result,expected", CASES)
59
+ async def test_async_tool_calls_async_sdk(cls, tool_input, method, args, kwargs, result, expected):
60
+ client, aclient = _clients()
61
+ setattr(aclient, method, AsyncMock(return_value=result))
62
+ tool = cls(client=client, async_client=aclient)
63
+ assert await tool.ainvoke(tool_input) == expected
64
+ getattr(aclient, method).assert_awaited_once_with(*args, **kwargs)
65
+ getattr(client, method).assert_not_called()
66
+
67
+
68
+ async def test_async_falls_back_to_sync_client():
69
+ client = MagicMock(spec=ClawFetch)
70
+ client.fetch.return_value = PAGE
71
+ tool = ClawFetchFetch(client=client)
72
+ assert await tool.ainvoke({"url": "https://a.b"}) == "# T\nURL: https://a.b\n\nhello"
73
+
74
+
75
+ def test_output_truncated_to_max_chars():
76
+ client, aclient = _clients()
77
+ client.parse.return_value = {"markdown": "x" * 50}
78
+ out = ClawFetchParse(client=client, async_client=aclient, max_chars=10).invoke({"url": "https://a.b/x.pdf"})
79
+ assert out == "x" * 10 + "\n...[truncated 40 chars]"
80
+
81
+
82
+ def test_sdk_error_becomes_tool_exception():
83
+ client, aclient = _clients()
84
+ client.research.side_effect = PaymentError("insufficient USDC", endpoint="/research")
85
+ with pytest.raises(ToolException, match="clawfetch_research failed \\(PaymentError\\): insufficient USDC"):
86
+ ClawFetchResearch(client=client, async_client=aclient).invoke({"query": "x"})
87
+
88
+
89
+ def test_sdk_error_returned_as_text_when_handle_tool_error():
90
+ client, aclient = _clients()
91
+ client.fetch.side_effect = PaymentError("insufficient USDC", endpoint="/fetch")
92
+ tool = ClawFetchFetch(client=client, async_client=aclient, handle_tool_error=True)
93
+ assert "insufficient USDC" in tool.invoke({"url": "https://a.b"})
94
+
95
+
96
+ def test_tool_metadata():
97
+ client, aclient = _clients()
98
+ tools = ClawFetchToolkit(client=client, async_client=aclient).get_tools()
99
+ assert [t.name for t in tools] == [
100
+ "clawfetch_fetch", "clawfetch_render", "clawfetch_extract", "clawfetch_research",
101
+ "clawfetch_parse", "clawfetch_domains_check", "clawfetch_domains_suggest",
102
+ ]
103
+ for t in tools:
104
+ assert "$0." in t.description and "USDC" in t.description
105
+ assert t.client is client and t.async_client is aclient
106
+
107
+
108
+ def test_toolkit_from_private_key():
109
+ tools = ClawFetchToolkit(private_key=DUMMY_KEY, max_chars=500).get_tools()
110
+ assert isinstance(tools[0].client, ClawFetch)
111
+ assert isinstance(tools[0].async_client, AsyncClawFetch)
112
+ assert tools[0].client is tools[-1].client
113
+ assert all(t.max_chars == 500 for t in tools)
114
+
115
+
116
+ def test_toolkit_from_env(monkeypatch):
117
+ monkeypatch.setenv("CLAWFETCH_PRIVATE_KEY", DUMMY_KEY)
118
+ assert isinstance(ClawFetchFetch().client, ClawFetch)
119
+
120
+
121
+ def test_missing_key_raises(monkeypatch):
122
+ monkeypatch.delenv("CLAWFETCH_PRIVATE_KEY", raising=False)
123
+ with pytest.raises(ValueError, match="CLAWFETCH_PRIVATE_KEY"):
124
+ ClawFetchToolkit()
125
+
126
+
127
+ def test_private_key_not_serialized():
128
+ tool = ClawFetchFetch(private_key=DUMMY_KEY)
129
+ assert DUMMY_KEY not in repr(tool) and DUMMY_KEY not in str(tool.model_dump())