langchain-x402-cleanweb 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langchain_x402_cleanweb-0.1.0/PKG-INFO +127 -0
- langchain_x402_cleanweb-0.1.0/README.md +103 -0
- langchain_x402_cleanweb-0.1.0/langchain_x402_cleanweb/__init__.py +10 -0
- langchain_x402_cleanweb-0.1.0/langchain_x402_cleanweb/py.typed +1 -0
- langchain_x402_cleanweb-0.1.0/langchain_x402_cleanweb/tools.py +163 -0
- langchain_x402_cleanweb-0.1.0/langchain_x402_cleanweb.egg-info/PKG-INFO +127 -0
- langchain_x402_cleanweb-0.1.0/langchain_x402_cleanweb.egg-info/SOURCES.txt +11 -0
- langchain_x402_cleanweb-0.1.0/langchain_x402_cleanweb.egg-info/dependency_links.txt +1 -0
- langchain_x402_cleanweb-0.1.0/langchain_x402_cleanweb.egg-info/requires.txt +3 -0
- langchain_x402_cleanweb-0.1.0/langchain_x402_cleanweb.egg-info/top_level.txt +1 -0
- langchain_x402_cleanweb-0.1.0/pyproject.toml +51 -0
- langchain_x402_cleanweb-0.1.0/setup.cfg +4 -0
- langchain_x402_cleanweb-0.1.0/tests/test_tools.py +79 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: langchain-x402-cleanweb
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: LangChain official tool integration for x402 CleanWeb Agent (87% token reduction, multimodal alt-text, anti-prompt injection, and autonomous micropayments)
|
|
5
|
+
Author-email: nohosa001-pixel <nohosa001@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/nohosa001-pixel/x402-cleanweb-agent
|
|
8
|
+
Project-URL: Repository, https://github.com/nohosa001-pixel/langchain-x402-cleanweb
|
|
9
|
+
Project-URL: Documentation, https://x402-cleanweb-agent-7qxtp3324q-du.a.run.app/docs
|
|
10
|
+
Keywords: langchain,langchain-tool,langchain-community,web-scraping,token-reduction,prompt-injection,cleanweb,x402,solana,base,polygon
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Requires-Python: >=3.9
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
Requires-Dist: requests>=2.31.0
|
|
22
|
+
Requires-Dist: pydantic>=2.0
|
|
23
|
+
Requires-Dist: langchain-core>=0.1.0
|
|
24
|
+
|
|
25
|
+
# LangChain x402 CleanWeb Integration
|
|
26
|
+
|
|
27
|
+
Official LangChain tool integration for **x402 CleanWeb Studio** (`x402-cleanweb-agent`).
|
|
28
|
+
|
|
29
|
+
Enables autonomous LangChain agents, conversational bots, and RAG pipelines to scrape and ingest web content at scale without drowning LLM context windows in boilerplate HTML, navigation scripts, tracking pixels, and ads.
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## ⚡ Key Features
|
|
34
|
+
|
|
35
|
+
- **87% Token Compression**: Strips boilerplate, ads, cookie modals, and navigation markup before index embedding.
|
|
36
|
+
- **Indirect Prompt Injection Defense**: Neutralizes hidden CSS text, zero-width steganography, and malicious query-parameter exfiltration links.
|
|
37
|
+
- **Multimodal Alt-Text Preservation**: Extracts descriptive image `alt` tags (``) for visual LLMs while neutralizing 1x1 tracking pixels.
|
|
38
|
+
- **SPA Hydration Extraction**: Zero-overhead pre-rendered state parsing for Next.js (`__NEXT_DATA__`), Nuxt.js (`__NUXT_DATA__`), and Schema.org JSON-LD.
|
|
39
|
+
- **x402 Autonomous Micropayments**: Native machine-to-machine USDC micropayments over Base, Polygon, Arbitrum, and Solana with a zero-auth 5-call trial tier.
|
|
40
|
+
- **LangChain Standard Compliance**: Implements both sync `_run` and async `_arun` via `langchain_core.tools.BaseTool` with Pydantic v2 `args_schema`.
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## 📦 Installation
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install langchain-x402-cleanweb
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## 🚀 Quickstart
|
|
53
|
+
|
|
54
|
+
### 1. Direct Tool Invocation
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from langchain_x402_cleanweb import X402CleanWebTool
|
|
58
|
+
|
|
59
|
+
# Initialize tool (uses official Cloud Run gateway by default)
|
|
60
|
+
tool = X402CleanWebTool()
|
|
61
|
+
|
|
62
|
+
# Scrape and clean web page
|
|
63
|
+
clean_markdown = tool.run("https://en.wikipedia.org/wiki/Web_scraping")
|
|
64
|
+
print(clean_markdown)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
### 2. Autonomous LangChain Agent Integration
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from langchain.agents import initialize_agent, AgentType
|
|
71
|
+
from langchain_openai import ChatOpenAI
|
|
72
|
+
from langchain_x402_cleanweb import X402CleanWebTool
|
|
73
|
+
|
|
74
|
+
llm = ChatOpenAI(model="gpt-4o", temperature=0)
|
|
75
|
+
tools = [X402CleanWebTool()]
|
|
76
|
+
|
|
77
|
+
agent = initialize_agent(
|
|
78
|
+
tools=tools,
|
|
79
|
+
llm=llm,
|
|
80
|
+
agent=AgentType.STRUCTURED_CHAT_ZERO_SHOT_REACT_DESCRIPTION,
|
|
81
|
+
verbose=True
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
response = agent.run("Summarize the latest engineering news on https://news.ycombinator.com")
|
|
85
|
+
print(response)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### 3. Asynchronous Execution
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
import asyncio
|
|
92
|
+
from langchain_x402_cleanweb import X402CleanWebTool
|
|
93
|
+
|
|
94
|
+
async def main():
|
|
95
|
+
tool = X402CleanWebTool()
|
|
96
|
+
content = await tool.arun("https://docs.python.org/3/whatsnew/3.13.html")
|
|
97
|
+
print(content[:500])
|
|
98
|
+
|
|
99
|
+
asyncio.run(main())
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## ⚙️ Configuration & Authentication
|
|
105
|
+
|
|
106
|
+
The tool works out of the box with **5 free trial requests** per session nonce.
|
|
107
|
+
|
|
108
|
+
For production high-throughput autonomous agents, set the environment variable:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
export AGENT_VAULT_KEY="your_funded_vault_session_key"
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Or pass it programmatically:
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
tool = X402CleanWebTool(
|
|
118
|
+
vault_key="your_vault_key",
|
|
119
|
+
gateway_url="https://x402-cleanweb-agent-7qxtp3324q-du.a.run.app"
|
|
120
|
+
)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## 🛡️ License
|
|
126
|
+
|
|
127
|
+
MIT License. Developed by the x402 Autonomous Agent Ecosystem.
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# LangChain x402 CleanWeb Integration
|
|
2
|
+
|
|
3
|
+
Official LangChain tool integration for **x402 CleanWeb Studio** (`x402-cleanweb-agent`).
|
|
4
|
+
|
|
5
|
+
Enables autonomous LangChain agents, conversational bots, and RAG pipelines to scrape and ingest web content at scale without drowning LLM context windows in boilerplate HTML, navigation scripts, tracking pixels, and ads.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## ⚡ Key Features
|
|
10
|
+
|
|
11
|
+
- **87% Token Compression**: Strips boilerplate, ads, cookie modals, and navigation markup before index embedding.
|
|
12
|
+
- **Indirect Prompt Injection Defense**: Neutralizes hidden CSS text, zero-width steganography, and malicious query-parameter exfiltration links.
|
|
13
|
+
- **Multimodal Alt-Text Preservation**: Extracts descriptive image `alt` tags (``) for visual LLMs while neutralizing 1x1 tracking pixels.
|
|
14
|
+
- **SPA Hydration Extraction**: Zero-overhead pre-rendered state parsing for Next.js (`__NEXT_DATA__`), Nuxt.js (`__NUXT_DATA__`), and Schema.org JSON-LD.
|
|
15
|
+
- **x402 Autonomous Micropayments**: Native machine-to-machine USDC micropayments over Base, Polygon, Arbitrum, and Solana with a zero-auth 5-call trial tier.
|
|
16
|
+
- **LangChain Standard Compliance**: Implements both sync `_run` and async `_arun` via `langchain_core.tools.BaseTool` with Pydantic v2 `args_schema`.
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
## 📦 Installation
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install langchain-x402-cleanweb
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
## 🚀 Quickstart
|
|
29
|
+
|
|
30
|
+
### 1. Direct Tool Invocation
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from langchain_x402_cleanweb import X402CleanWebTool
|
|
34
|
+
|
|
35
|
+
# Initialize tool (uses official Cloud Run gateway by default)
|
|
36
|
+
tool = X402CleanWebTool()
|
|
37
|
+
|
|
38
|
+
# Scrape and clean web page
|
|
39
|
+
clean_markdown = tool.run("https://en.wikipedia.org/wiki/Web_scraping")
|
|
40
|
+
print(clean_markdown)
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
### 2. Autonomous LangChain Agent Integration
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from langchain.agents import initialize_agent, AgentType
|
|
47
|
+
from langchain_openai import ChatOpenAI
|
|
48
|
+
from langchain_x402_cleanweb import X402CleanWebTool
|
|
49
|
+
|
|
50
|
+
llm = ChatOpenAI(model="gpt-4o", temperature=0)
|
|
51
|
+
tools = [X402CleanWebTool()]
|
|
52
|
+
|
|
53
|
+
agent = initialize_agent(
|
|
54
|
+
tools=tools,
|
|
55
|
+
llm=llm,
|
|
56
|
+
agent=AgentType.STRUCTURED_CHAT_ZERO_SHOT_REACT_DESCRIPTION,
|
|
57
|
+
verbose=True
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
response = agent.run("Summarize the latest engineering news on https://news.ycombinator.com")
|
|
61
|
+
print(response)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
### 3. Asynchronous Execution
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
import asyncio
|
|
68
|
+
from langchain_x402_cleanweb import X402CleanWebTool
|
|
69
|
+
|
|
70
|
+
async def main():
|
|
71
|
+
tool = X402CleanWebTool()
|
|
72
|
+
content = await tool.arun("https://docs.python.org/3/whatsnew/3.13.html")
|
|
73
|
+
print(content[:500])
|
|
74
|
+
|
|
75
|
+
asyncio.run(main())
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
---
|
|
79
|
+
|
|
80
|
+
## ⚙️ Configuration & Authentication
|
|
81
|
+
|
|
82
|
+
The tool works out of the box with **5 free trial requests** per session nonce.
|
|
83
|
+
|
|
84
|
+
For production high-throughput autonomous agents, set the environment variable:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
export AGENT_VAULT_KEY="your_funded_vault_session_key"
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Or pass it programmatically:
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
tool = X402CleanWebTool(
|
|
94
|
+
vault_key="your_vault_key",
|
|
95
|
+
gateway_url="https://x402-cleanweb-agent-7qxtp3324q-du.a.run.app"
|
|
96
|
+
)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## 🛡️ License
|
|
102
|
+
|
|
103
|
+
MIT License. Developed by the x402 Autonomous Agent Ecosystem.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""
|
|
2
|
+
LangChain x402 CleanWeb Tool Integration Package
|
|
3
|
+
================================================
|
|
4
|
+
Autonomous 87% token-reduced web scraping with prompt injection defense & x402 micropayments.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from langchain_x402_cleanweb.tools import X402CleanWebTool, X402CleanWebInput
|
|
8
|
+
|
|
9
|
+
__all__ = ["X402CleanWebTool", "X402CleanWebInput"]
|
|
10
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Marker file for PEP 561.
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""
|
|
2
|
+
LangChain Official Tool Specification for x402 CleanWeb Agent
|
|
3
|
+
--------------------------------------------------------------------------------
|
|
4
|
+
Enables autonomous AI agents to fetch clean, noise-free Markdown with 87% token
|
|
5
|
+
savings and zero credit card dependency via x402 micropayments.
|
|
6
|
+
|
|
7
|
+
Complies with LangChain standard BaseTool specification (sync _run, async _arun, Pydantic v2 args_schema).
|
|
8
|
+
Supports v2.8.4 Multimodal Alt-Text Preservation and Zero-Overhead SPA Hydration.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
import secrets
|
|
15
|
+
import time
|
|
16
|
+
from typing import Any, Dict, Optional, Type
|
|
17
|
+
|
|
18
|
+
import requests
|
|
19
|
+
from pydantic import BaseModel, Field
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
from langchain_core.tools import BaseTool
|
|
23
|
+
from langchain_core.callbacks import CallbackManagerForToolRun, AsyncCallbackManagerForToolRun
|
|
24
|
+
except ImportError:
|
|
25
|
+
# Fallback dummy class adhering to Pydantic BaseModel if langchain_core is not installed
|
|
26
|
+
class BaseTool(BaseModel): # type: ignore
|
|
27
|
+
name: str = ""
|
|
28
|
+
description: str = ""
|
|
29
|
+
args_schema: Optional[Type[BaseModel]] = None
|
|
30
|
+
CallbackManagerForToolRun = Any # type: ignore
|
|
31
|
+
AsyncCallbackManagerForToolRun = Any # type: ignore
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class X402CleanWebInput(BaseModel):
|
|
35
|
+
"""Input parameters for the x402 CleanWeb tool."""
|
|
36
|
+
url: str = Field(
|
|
37
|
+
...,
|
|
38
|
+
description="The exact HTTP/HTTPS URL of the webpage to scrape, strip ads and boilerplate from, and convert to clean Markdown."
|
|
39
|
+
)
|
|
40
|
+
raw_markdown: bool = Field(
|
|
41
|
+
default=True,
|
|
42
|
+
description="Whether to return pure stripped markdown optimized for LLM token savings."
|
|
43
|
+
)
|
|
44
|
+
preserve_alt_text: bool = Field(
|
|
45
|
+
default=True,
|
|
46
|
+
description="Preserve descriptive image alt text () for multimodal LLMs while neutralizing covert tracking pixels."
|
|
47
|
+
)
|
|
48
|
+
dynamic_render: bool = Field(
|
|
49
|
+
default=False,
|
|
50
|
+
description="Enable client-side JavaScript rendering and SPA hydration extraction for single-page apps."
|
|
51
|
+
)
|
|
52
|
+
scroll_depth: int = Field(
|
|
53
|
+
default=0,
|
|
54
|
+
description="Number of infinite scroll iterations to trigger dynamic lazy-loaded content."
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class X402CleanWebTool(BaseTool):
|
|
59
|
+
"""
|
|
60
|
+
Tool that uses the x402 CleanWeb Gateway to scrape webpages,
|
|
61
|
+
stripping 87% of HTML boilerplate, ads, and noisy scripts, returning
|
|
62
|
+
token-optimized clean Markdown with autonomous Web3 micropayment support.
|
|
63
|
+
"""
|
|
64
|
+
name: str = "x402_clean_web"
|
|
65
|
+
description: str = (
|
|
66
|
+
"A high-throughput web scraping and cleaning tool. "
|
|
67
|
+
"Transforms cluttered HTML webpages into 87% token-reduced clean Markdown. "
|
|
68
|
+
"Supports multimodal image alt-text preservation and SPA hydration extraction. "
|
|
69
|
+
"Useful for web research, reading articles, documentation, or news without ad noise. "
|
|
70
|
+
"Input should be a valid web URL."
|
|
71
|
+
)
|
|
72
|
+
args_schema: Type[BaseModel] = X402CleanWebInput
|
|
73
|
+
|
|
74
|
+
gateway_url: str = Field(
|
|
75
|
+
default_factory=lambda: os.getenv("X402_GATEWAY_URL", "https://x402-cleanweb-agent-7qxtp3324q-du.a.run.app")
|
|
76
|
+
)
|
|
77
|
+
vault_key: Optional[str] = Field(
|
|
78
|
+
default_factory=lambda: os.getenv("AGENT_VAULT_KEY")
|
|
79
|
+
)
|
|
80
|
+
timeout: int = Field(default=25)
|
|
81
|
+
|
|
82
|
+
def _build_headers(self) -> Dict[str, str]:
|
|
83
|
+
session_nonce = f"langchain_{secrets.token_hex(4)}_{int(time.time())}"
|
|
84
|
+
headers = {
|
|
85
|
+
"User-Agent": "LangChain-x402-CleanWeb/1.0",
|
|
86
|
+
"X-Agent-Nonce": session_nonce,
|
|
87
|
+
"Accept": "application/json"
|
|
88
|
+
}
|
|
89
|
+
if self.vault_key:
|
|
90
|
+
headers["X-Agent-Pass"] = self.vault_key
|
|
91
|
+
headers["X-Vault-Key"] = self.vault_key
|
|
92
|
+
return headers
|
|
93
|
+
|
|
94
|
+
def _run(
|
|
95
|
+
self,
|
|
96
|
+
url: str,
|
|
97
|
+
raw_markdown: bool = True,
|
|
98
|
+
preserve_alt_text: bool = True,
|
|
99
|
+
dynamic_render: bool = False,
|
|
100
|
+
scroll_depth: int = 0,
|
|
101
|
+
run_manager: Optional[CallbackManagerForToolRun] = None,
|
|
102
|
+
) -> str:
|
|
103
|
+
"""Synchronously execute the web clean request."""
|
|
104
|
+
try:
|
|
105
|
+
headers = self._build_headers()
|
|
106
|
+
params = {
|
|
107
|
+
"url": url,
|
|
108
|
+
"preserve_alt_text": preserve_alt_text,
|
|
109
|
+
"dynamic_render": dynamic_render,
|
|
110
|
+
"scroll_depth": scroll_depth
|
|
111
|
+
}
|
|
112
|
+
response = requests.get(
|
|
113
|
+
f"{self.gateway_url.rstrip('/')}/api/v1/clean-web",
|
|
114
|
+
params=params,
|
|
115
|
+
headers=headers,
|
|
116
|
+
timeout=self.timeout
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
if response.status_code == 200:
|
|
120
|
+
data = response.json()
|
|
121
|
+
title = data.get("title", "Clean Web Result")
|
|
122
|
+
content = data.get("clean_markdown") or data.get("markdown_content") or data.get("content", "")
|
|
123
|
+
tokens_saved = data.get("estimated_token_savings") or data.get("token_analytics", {}).get("savings_percentage", "87%")
|
|
124
|
+
|
|
125
|
+
return (
|
|
126
|
+
f"### {title}\n"
|
|
127
|
+
f"*(Extracted via x402 CleanWeb Gateway — Token Savings: {tokens_saved})*\n\n"
|
|
128
|
+
f"{content}"
|
|
129
|
+
)
|
|
130
|
+
elif response.status_code == 402:
|
|
131
|
+
challenge = response.json() if response.headers.get("content-type") == "application/json" else {}
|
|
132
|
+
return (
|
|
133
|
+
f"[x402 Payment Required]: Free trial quota exhausted. "
|
|
134
|
+
f"Please pre-fund your agent vault on Solana/Polygon/Base or set AGENT_VAULT_KEY. "
|
|
135
|
+
f"Challenge Details: {challenge}"
|
|
136
|
+
)
|
|
137
|
+
else:
|
|
138
|
+
return f"[x402 CleanWeb Error]: HTTP {response.status_code} - {response.text[:300]}"
|
|
139
|
+
|
|
140
|
+
except requests.exceptions.Timeout:
|
|
141
|
+
return f"[x402 CleanWeb Error]: Request timed out after {self.timeout}s for URL: {url}"
|
|
142
|
+
except Exception as e:
|
|
143
|
+
return f"[x402 CleanWeb Error]: Unexpected exception: {str(e)}"
|
|
144
|
+
|
|
145
|
+
async def _arun(
|
|
146
|
+
self,
|
|
147
|
+
url: str,
|
|
148
|
+
raw_markdown: bool = True,
|
|
149
|
+
preserve_alt_text: bool = True,
|
|
150
|
+
dynamic_render: bool = False,
|
|
151
|
+
scroll_depth: int = 0,
|
|
152
|
+
run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
|
|
153
|
+
) -> str:
|
|
154
|
+
"""Asynchronously execute the web clean request."""
|
|
155
|
+
import asyncio
|
|
156
|
+
return await asyncio.to_thread(
|
|
157
|
+
self._run,
|
|
158
|
+
url,
|
|
159
|
+
raw_markdown,
|
|
160
|
+
preserve_alt_text,
|
|
161
|
+
dynamic_render,
|
|
162
|
+
scroll_depth
|
|
163
|
+
)
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: langchain-x402-cleanweb
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: LangChain official tool integration for x402 CleanWeb Agent (87% token reduction, multimodal alt-text, anti-prompt injection, and autonomous micropayments)
|
|
5
|
+
Author-email: nohosa001-pixel <nohosa001@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/nohosa001-pixel/x402-cleanweb-agent
|
|
8
|
+
Project-URL: Repository, https://github.com/nohosa001-pixel/langchain-x402-cleanweb
|
|
9
|
+
Project-URL: Documentation, https://x402-cleanweb-agent-7qxtp3324q-du.a.run.app/docs
|
|
10
|
+
Keywords: langchain,langchain-tool,langchain-community,web-scraping,token-reduction,prompt-injection,cleanweb,x402,solana,base,polygon
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Requires-Python: >=3.9
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
Requires-Dist: requests>=2.31.0
|
|
22
|
+
Requires-Dist: pydantic>=2.0
|
|
23
|
+
Requires-Dist: langchain-core>=0.1.0
|
|
24
|
+
|
|
25
|
+
# LangChain x402 CleanWeb Integration
|
|
26
|
+
|
|
27
|
+
Official LangChain tool integration for **x402 CleanWeb Studio** (`x402-cleanweb-agent`).
|
|
28
|
+
|
|
29
|
+
Enables autonomous LangChain agents, conversational bots, and RAG pipelines to scrape and ingest web content at scale without drowning LLM context windows in boilerplate HTML, navigation scripts, tracking pixels, and ads.
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## ⚡ Key Features
|
|
34
|
+
|
|
35
|
+
- **87% Token Compression**: Strips boilerplate, ads, cookie modals, and navigation markup before index embedding.
|
|
36
|
+
- **Indirect Prompt Injection Defense**: Neutralizes hidden CSS text, zero-width steganography, and malicious query-parameter exfiltration links.
|
|
37
|
+
- **Multimodal Alt-Text Preservation**: Extracts descriptive image `alt` tags (``) for visual LLMs while neutralizing 1x1 tracking pixels.
|
|
38
|
+
- **SPA Hydration Extraction**: Zero-overhead pre-rendered state parsing for Next.js (`__NEXT_DATA__`), Nuxt.js (`__NUXT_DATA__`), and Schema.org JSON-LD.
|
|
39
|
+
- **x402 Autonomous Micropayments**: Native machine-to-machine USDC micropayments over Base, Polygon, Arbitrum, and Solana with a zero-auth 5-call trial tier.
|
|
40
|
+
- **LangChain Standard Compliance**: Implements both sync `_run` and async `_arun` via `langchain_core.tools.BaseTool` with Pydantic v2 `args_schema`.
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## 📦 Installation
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install langchain-x402-cleanweb
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## 🚀 Quickstart
|
|
53
|
+
|
|
54
|
+
### 1. Direct Tool Invocation
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from langchain_x402_cleanweb import X402CleanWebTool
|
|
58
|
+
|
|
59
|
+
# Initialize tool (uses official Cloud Run gateway by default)
|
|
60
|
+
tool = X402CleanWebTool()
|
|
61
|
+
|
|
62
|
+
# Scrape and clean web page
|
|
63
|
+
clean_markdown = tool.run("https://en.wikipedia.org/wiki/Web_scraping")
|
|
64
|
+
print(clean_markdown)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
### 2. Autonomous LangChain Agent Integration
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from langchain.agents import initialize_agent, AgentType
|
|
71
|
+
from langchain_openai import ChatOpenAI
|
|
72
|
+
from langchain_x402_cleanweb import X402CleanWebTool
|
|
73
|
+
|
|
74
|
+
llm = ChatOpenAI(model="gpt-4o", temperature=0)
|
|
75
|
+
tools = [X402CleanWebTool()]
|
|
76
|
+
|
|
77
|
+
agent = initialize_agent(
|
|
78
|
+
tools=tools,
|
|
79
|
+
llm=llm,
|
|
80
|
+
agent=AgentType.STRUCTURED_CHAT_ZERO_SHOT_REACT_DESCRIPTION,
|
|
81
|
+
verbose=True
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
response = agent.run("Summarize the latest engineering news on https://news.ycombinator.com")
|
|
85
|
+
print(response)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### 3. Asynchronous Execution
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
import asyncio
|
|
92
|
+
from langchain_x402_cleanweb import X402CleanWebTool
|
|
93
|
+
|
|
94
|
+
async def main():
|
|
95
|
+
tool = X402CleanWebTool()
|
|
96
|
+
content = await tool.arun("https://docs.python.org/3/whatsnew/3.13.html")
|
|
97
|
+
print(content[:500])
|
|
98
|
+
|
|
99
|
+
asyncio.run(main())
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## ⚙️ Configuration & Authentication
|
|
105
|
+
|
|
106
|
+
The tool works out of the box with **5 free trial requests** per session nonce.
|
|
107
|
+
|
|
108
|
+
For production high-throughput autonomous agents, set the environment variable:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
export AGENT_VAULT_KEY="your_funded_vault_session_key"
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Or pass it programmatically:
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
tool = X402CleanWebTool(
|
|
118
|
+
vault_key="your_vault_key",
|
|
119
|
+
gateway_url="https://x402-cleanweb-agent-7qxtp3324q-du.a.run.app"
|
|
120
|
+
)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## 🛡️ License
|
|
126
|
+
|
|
127
|
+
MIT License. Developed by the x402 Autonomous Agent Ecosystem.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
langchain_x402_cleanweb/__init__.py
|
|
4
|
+
langchain_x402_cleanweb/py.typed
|
|
5
|
+
langchain_x402_cleanweb/tools.py
|
|
6
|
+
langchain_x402_cleanweb.egg-info/PKG-INFO
|
|
7
|
+
langchain_x402_cleanweb.egg-info/SOURCES.txt
|
|
8
|
+
langchain_x402_cleanweb.egg-info/dependency_links.txt
|
|
9
|
+
langchain_x402_cleanweb.egg-info/requires.txt
|
|
10
|
+
langchain_x402_cleanweb.egg-info/top_level.txt
|
|
11
|
+
tests/test_tools.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
langchain_x402_cleanweb
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "langchain-x402-cleanweb"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "LangChain official tool integration for x402 CleanWeb Agent (87% token reduction, multimodal alt-text, anti-prompt injection, and autonomous micropayments)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "nohosa001-pixel", email = "nohosa001@gmail.com" }
|
|
14
|
+
]
|
|
15
|
+
keywords = [
|
|
16
|
+
"langchain",
|
|
17
|
+
"langchain-tool",
|
|
18
|
+
"langchain-community",
|
|
19
|
+
"web-scraping",
|
|
20
|
+
"token-reduction",
|
|
21
|
+
"prompt-injection",
|
|
22
|
+
"cleanweb",
|
|
23
|
+
"x402",
|
|
24
|
+
"solana",
|
|
25
|
+
"base",
|
|
26
|
+
"polygon"
|
|
27
|
+
]
|
|
28
|
+
classifiers = [
|
|
29
|
+
"Development Status :: 5 - Production/Stable",
|
|
30
|
+
"Intended Audience :: Developers",
|
|
31
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
32
|
+
"Programming Language :: Python :: 3",
|
|
33
|
+
"Programming Language :: Python :: 3.9",
|
|
34
|
+
"Programming Language :: Python :: 3.10",
|
|
35
|
+
"Programming Language :: Python :: 3.11",
|
|
36
|
+
"Programming Language :: Python :: 3.12"
|
|
37
|
+
]
|
|
38
|
+
dependencies = [
|
|
39
|
+
"requests>=2.31.0",
|
|
40
|
+
"pydantic>=2.0",
|
|
41
|
+
"langchain-core>=0.1.0"
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[tool.setuptools.packages.find]
|
|
45
|
+
where = ["."]
|
|
46
|
+
include = ["langchain_x402_cleanweb*"]
|
|
47
|
+
|
|
48
|
+
[project.urls]
|
|
49
|
+
Homepage = "https://github.com/nohosa001-pixel/x402-cleanweb-agent"
|
|
50
|
+
Repository = "https://github.com/nohosa001-pixel/langchain-x402-cleanweb"
|
|
51
|
+
Documentation = "https://x402-cleanweb-agent-7qxtp3324q-du.a.run.app/docs"
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Unit tests for langchain-x402-cleanweb package
|
|
3
|
+
----------------------------------------------
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import unittest
|
|
7
|
+
from unittest.mock import patch, MagicMock
|
|
8
|
+
import asyncio
|
|
9
|
+
|
|
10
|
+
from langchain_x402_cleanweb import X402CleanWebTool, X402CleanWebInput
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class TestX402CleanWebTool(unittest.TestCase):
|
|
14
|
+
def setUp(self):
|
|
15
|
+
self.tool = X402CleanWebTool(gateway_url="https://mock-gateway.local", vault_key="TEST_KEY")
|
|
16
|
+
|
|
17
|
+
def test_tool_metadata(self):
|
|
18
|
+
"""Verify tool adheres to LangChain naming and schema standards."""
|
|
19
|
+
self.assertEqual(self.tool.name, "x402_clean_web")
|
|
20
|
+
self.assertTrue("token-reduced clean Markdown" in self.tool.description)
|
|
21
|
+
self.assertEqual(self.tool.args_schema, X402CleanWebInput)
|
|
22
|
+
|
|
23
|
+
@patch("requests.get")
|
|
24
|
+
def test_run_success_200(self, mock_get):
|
|
25
|
+
"""Verify standard 200 OK clean web extraction."""
|
|
26
|
+
mock_response = MagicMock()
|
|
27
|
+
mock_response.status_code = 200
|
|
28
|
+
mock_response.json.return_value = {
|
|
29
|
+
"title": "Sample Article",
|
|
30
|
+
"clean_markdown": "# Great Work\nHere is the essence of great work.",
|
|
31
|
+
"estimated_token_savings": "88.5%"
|
|
32
|
+
}
|
|
33
|
+
mock_get.return_value = mock_response
|
|
34
|
+
|
|
35
|
+
result = self.tool._run(url="https://example.com/article")
|
|
36
|
+
self.assertIn("Sample Article", result)
|
|
37
|
+
self.assertIn("88.5%", result)
|
|
38
|
+
self.assertIn("# Great Work", result)
|
|
39
|
+
|
|
40
|
+
@patch("requests.get")
|
|
41
|
+
def test_run_payment_required_402(self, mock_get):
|
|
42
|
+
"""Verify proper handling of HTTP 402 challenge."""
|
|
43
|
+
mock_response = MagicMock()
|
|
44
|
+
mock_response.status_code = 402
|
|
45
|
+
mock_response.headers = {"content-type": "application/json"}
|
|
46
|
+
mock_response.json.return_value = {
|
|
47
|
+
"error": "Payment Required",
|
|
48
|
+
"price_usdc": 0.001,
|
|
49
|
+
"recipient": "411ksMz9RHYVtVMe6RUUErzZYtrU9zzvkgzswKbqx9qp"
|
|
50
|
+
}
|
|
51
|
+
mock_get.return_value = mock_response
|
|
52
|
+
|
|
53
|
+
result = self.tool._run(url="https://example.com/paid-article")
|
|
54
|
+
self.assertIn("[x402 Payment Required]", result)
|
|
55
|
+
self.assertIn("Free trial quota exhausted", result)
|
|
56
|
+
|
|
57
|
+
@patch("requests.get")
|
|
58
|
+
def test_arun_invocation(self, mock_get):
|
|
59
|
+
"""Verify asynchronous invocation works properly."""
|
|
60
|
+
mock_response = MagicMock()
|
|
61
|
+
mock_response.status_code = 200
|
|
62
|
+
mock_response.json.return_value = {
|
|
63
|
+
"title": "Async Article",
|
|
64
|
+
"clean_markdown": "# Async Content",
|
|
65
|
+
"estimated_token_savings": "90%"
|
|
66
|
+
}
|
|
67
|
+
mock_get.return_value = mock_response
|
|
68
|
+
|
|
69
|
+
loop = asyncio.new_event_loop()
|
|
70
|
+
try:
|
|
71
|
+
result = loop.run_until_complete(self.tool._arun(url="https://example.com"))
|
|
72
|
+
self.assertIn("Async Article", result)
|
|
73
|
+
self.assertIn("# Async Content", result)
|
|
74
|
+
finally:
|
|
75
|
+
loop.close()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
if __name__ == "__main__":
|
|
79
|
+
unittest.main()
|