karon-mcp 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- karon_mcp-1.0.0/.gitignore +42 -0
- karon_mcp-1.0.0/CHANGELOG.md +15 -0
- karon_mcp-1.0.0/LICENSE +21 -0
- karon_mcp-1.0.0/PKG-INFO +127 -0
- karon_mcp-1.0.0/README.md +100 -0
- karon_mcp-1.0.0/pyproject.toml +43 -0
- karon_mcp-1.0.0/src/karon_mcp/__init__.py +3 -0
- karon_mcp-1.0.0/src/karon_mcp/__main__.py +10 -0
- karon_mcp-1.0.0/src/karon_mcp/server.py +349 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Secrets — NEVER commit
|
|
2
|
+
.env
|
|
3
|
+
*.pem
|
|
4
|
+
*.key
|
|
5
|
+
|
|
6
|
+
# Build artifacts
|
|
7
|
+
/target/
|
|
8
|
+
|
|
9
|
+
# Logs
|
|
10
|
+
bot.log
|
|
11
|
+
build.log
|
|
12
|
+
*.log
|
|
13
|
+
|
|
14
|
+
# Backups
|
|
15
|
+
*.bak
|
|
16
|
+
*.bak_
|
|
17
|
+
/backups/
|
|
18
|
+
|
|
19
|
+
# OS / IDE
|
|
20
|
+
.DS_Store
|
|
21
|
+
.vscode/
|
|
22
|
+
.idea/
|
|
23
|
+
*.swp
|
|
24
|
+
*.swo
|
|
25
|
+
|
|
26
|
+
# Misc
|
|
27
|
+
/tmp/
|
|
28
|
+
target/
|
|
29
|
+
|
|
30
|
+
# OpenClaw runtime
|
|
31
|
+
openclaw-stack/node_modules/
|
|
32
|
+
openclaw-stack/profiles/
|
|
33
|
+
openclaw-stack/blitz_logs/
|
|
34
|
+
openclaw-stack/doctor/
|
|
35
|
+
openclaw-stack/package-lock.json
|
|
36
|
+
|
|
37
|
+
# Sensitive
|
|
38
|
+
.ssh/
|
|
39
|
+
wallet.json
|
|
40
|
+
ssh/
|
|
41
|
+
.env.bak*
|
|
42
|
+
*oauth*.json
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `karon-mcp` will be documented in this file.
|
|
4
|
+
|
|
5
|
+
## [1.0.0] - 2026-04-08
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
- `browse` tool — Fetch any URL bypassing WAFs (Cloudflare, DataDome, Kasada, Akamai, PerimeterX). Returns markdown, text, or raw HTML.
|
|
9
|
+
- `crawl` tool — Fetch up to 20 URLs concurrently with configurable parallelism (1-5).
|
|
10
|
+
- SSRF protection — Blocks requests to internal networks (RFC 1918, loopback, link-local, IPv4-mapped IPv6).
|
|
11
|
+
- DNS rebinding protection — Resolves hostnames and validates against blocked ranges.
|
|
12
|
+
- Response size limits — 10 MB max response, 200K content truncation.
|
|
13
|
+
- Global concurrency semaphore — Max 5 concurrent requests to protect upstream API.
|
|
14
|
+
- Singleton HTTP client with connection pooling.
|
|
15
|
+
- Control character escaping in all output strings.
|
karon_mcp-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Karon Labs
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
karon_mcp-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: karon-mcp
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Karon Web Unblocker — MCP server for Claude, Cursor, and other MCP clients. Fetch any URL bypassing Cloudflare, DataDome, Akamai, and other WAFs.
|
|
5
|
+
Project-URL: Homepage, https://karonlabs.net/api/
|
|
6
|
+
Project-URL: Documentation, https://karonlabs.net/api/docs.html
|
|
7
|
+
Project-URL: Repository, https://github.com/pplar39/karon3
|
|
8
|
+
Project-URL: Changelog, https://github.com/pplar39/karon3/blob/main/karon-mcp/CHANGELOG.md
|
|
9
|
+
Author-email: Karon Labs <ceo@karonlabs.net>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: claude,cloudflare,cursor,mcp,unblocker,waf-bypass,web-scraping
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Requires-Dist: httpx>=0.27.0
|
|
25
|
+
Requires-Dist: mcp>=1.0.0
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# karon-mcp
|
|
29
|
+
|
|
30
|
+
MCP server for the [Karon Web Unblocker API](https://karonlabs.net/api/). Fetch any URL bypassing Cloudflare, DataDome, Akamai, Kasada, and PerimeterX — directly from Claude, Cursor, or any MCP-compatible client.
|
|
31
|
+
|
|
32
|
+
## Quick Start
|
|
33
|
+
|
|
34
|
+
### 1. Get your API key
|
|
35
|
+
|
|
36
|
+
Sign up at [karonlabs.net/api/signup.html](https://karonlabs.net/api/signup.html) (Google login, free tier: 1,000 credits).
|
|
37
|
+
|
|
38
|
+
### 2. Install
|
|
39
|
+
|
|
40
|
+
**Claude Desktop / Claude Code** — add to your MCP config (`claude_desktop_config.json` or `.mcp.json`):
|
|
41
|
+
|
|
42
|
+
```json
|
|
43
|
+
{
|
|
44
|
+
"mcpServers": {
|
|
45
|
+
"karon": {
|
|
46
|
+
"command": "uvx",
|
|
47
|
+
"args": ["karon-mcp"],
|
|
48
|
+
"env": {
|
|
49
|
+
"KARON_API_KEY": "your_api_key_here"
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
> `uvx` auto-fetches the latest version from PyPI on every run. No manual updates needed.
|
|
57
|
+
|
|
58
|
+
**Cursor** — go to Settings > MCP Servers > Add, then use the same config above.
|
|
59
|
+
|
|
60
|
+
**Manual install** (if you prefer pip):
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install karon-mcp
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
### 3. Use
|
|
67
|
+
|
|
68
|
+
Once installed, two tools become available in your MCP client:
|
|
69
|
+
|
|
70
|
+
#### `browse` — Fetch a single URL
|
|
71
|
+
|
|
72
|
+
```
|
|
73
|
+
browse(url="https://example.com", extract="markdown", readability=True)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
| Parameter | Type | Default | Description |
|
|
77
|
+
|-----------|------|---------|-------------|
|
|
78
|
+
| `url` | string | required | Target URL (http/https) |
|
|
79
|
+
| `extract` | string | `"markdown"` | Output format: `"markdown"`, `"text"`, or `"html"` |
|
|
80
|
+
| `readability` | bool | `True` | `True` = main content only, `False` = full page |
|
|
81
|
+
|
|
82
|
+
#### `crawl` — Fetch multiple URLs concurrently
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
crawl(urls=["https://a.com", "https://b.com"], extract="markdown", concurrency=3)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
| Parameter | Type | Default | Description |
|
|
89
|
+
|-----------|------|---------|-------------|
|
|
90
|
+
| `urls` | list[string] | required | Up to 20 URLs |
|
|
91
|
+
| `extract` | string | `"markdown"` | Output format: `"markdown"` or `"text"` |
|
|
92
|
+
| `readability` | bool | `True` | Main content only |
|
|
93
|
+
| `concurrency` | int | `3` | Parallel requests (1-5) |
|
|
94
|
+
|
|
95
|
+
### Credits
|
|
96
|
+
|
|
97
|
+
| Method | Cost |
|
|
98
|
+
|--------|------|
|
|
99
|
+
| Cache hit | 1 credit |
|
|
100
|
+
| Browser render | 10 credits |
|
|
101
|
+
|
|
102
|
+
Free tier includes 1,000 credits. See [pricing](https://karonlabs.net/api/pricing.html) for paid plans.
|
|
103
|
+
|
|
104
|
+
## Configuration
|
|
105
|
+
|
|
106
|
+
The only required configuration is the `KARON_API_KEY` environment variable. Get yours at [karonlabs.net/api/signup.html](https://karonlabs.net/api/signup.html).
|
|
107
|
+
|
|
108
|
+
## Updating
|
|
109
|
+
|
|
110
|
+
If you're using `uvx` (recommended), updates are automatic — every run fetches the latest version.
|
|
111
|
+
|
|
112
|
+
If you installed with pip:
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
pip install --upgrade karon-mcp
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Links
|
|
119
|
+
|
|
120
|
+
- [API Documentation](https://karonlabs.net/api/docs.html)
|
|
121
|
+
- [Pricing](https://karonlabs.net/api/pricing.html)
|
|
122
|
+
- [WAF Compatibility](https://karonlabs.net/api/waf.html)
|
|
123
|
+
- [Changelog](CHANGELOG.md)
|
|
124
|
+
|
|
125
|
+
## License
|
|
126
|
+
|
|
127
|
+
MIT
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# karon-mcp
|
|
2
|
+
|
|
3
|
+
MCP server for the [Karon Web Unblocker API](https://karonlabs.net/api/). Fetch any URL bypassing Cloudflare, DataDome, Akamai, Kasada, and PerimeterX — directly from Claude, Cursor, or any MCP-compatible client.
|
|
4
|
+
|
|
5
|
+
## Quick Start
|
|
6
|
+
|
|
7
|
+
### 1. Get your API key
|
|
8
|
+
|
|
9
|
+
Sign up at [karonlabs.net/api/signup.html](https://karonlabs.net/api/signup.html) (Google login, free tier: 1,000 credits).
|
|
10
|
+
|
|
11
|
+
### 2. Install
|
|
12
|
+
|
|
13
|
+
**Claude Desktop / Claude Code** — add to your MCP config (`claude_desktop_config.json` or `.mcp.json`):
|
|
14
|
+
|
|
15
|
+
```json
|
|
16
|
+
{
|
|
17
|
+
"mcpServers": {
|
|
18
|
+
"karon": {
|
|
19
|
+
"command": "uvx",
|
|
20
|
+
"args": ["karon-mcp"],
|
|
21
|
+
"env": {
|
|
22
|
+
"KARON_API_KEY": "your_api_key_here"
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
> `uvx` auto-fetches the latest version from PyPI on every run. No manual updates needed.
|
|
30
|
+
|
|
31
|
+
**Cursor** — go to Settings > MCP Servers > Add, then use the same config above.
|
|
32
|
+
|
|
33
|
+
**Manual install** (if you prefer pip):
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install karon-mcp
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
### 3. Use
|
|
40
|
+
|
|
41
|
+
Once installed, two tools become available in your MCP client:
|
|
42
|
+
|
|
43
|
+
#### `browse` — Fetch a single URL
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
browse(url="https://example.com", extract="markdown", readability=True)
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
| Parameter | Type | Default | Description |
|
|
50
|
+
|-----------|------|---------|-------------|
|
|
51
|
+
| `url` | string | required | Target URL (http/https) |
|
|
52
|
+
| `extract` | string | `"markdown"` | Output format: `"markdown"`, `"text"`, or `"html"` |
|
|
53
|
+
| `readability` | bool | `True` | `True` = main content only, `False` = full page |
|
|
54
|
+
|
|
55
|
+
#### `crawl` — Fetch multiple URLs concurrently
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
crawl(urls=["https://a.com", "https://b.com"], extract="markdown", concurrency=3)
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
| Parameter | Type | Default | Description |
|
|
62
|
+
|-----------|------|---------|-------------|
|
|
63
|
+
| `urls` | list[string] | required | Up to 20 URLs |
|
|
64
|
+
| `extract` | string | `"markdown"` | Output format: `"markdown"` or `"text"` |
|
|
65
|
+
| `readability` | bool | `True` | Main content only |
|
|
66
|
+
| `concurrency` | int | `3` | Parallel requests (1-5) |
|
|
67
|
+
|
|
68
|
+
### Credits
|
|
69
|
+
|
|
70
|
+
| Method | Cost |
|
|
71
|
+
|--------|------|
|
|
72
|
+
| Cache hit | 1 credit |
|
|
73
|
+
| Browser render | 10 credits |
|
|
74
|
+
|
|
75
|
+
Free tier includes 1,000 credits. See [pricing](https://karonlabs.net/api/pricing.html) for paid plans.
|
|
76
|
+
|
|
77
|
+
## Configuration
|
|
78
|
+
|
|
79
|
+
The only required configuration is the `KARON_API_KEY` environment variable. Get yours at [karonlabs.net/api/signup.html](https://karonlabs.net/api/signup.html).
|
|
80
|
+
|
|
81
|
+
## Updating
|
|
82
|
+
|
|
83
|
+
If you're using `uvx` (recommended), updates are automatic — every run fetches the latest version.
|
|
84
|
+
|
|
85
|
+
If you installed with pip:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
pip install --upgrade karon-mcp
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Links
|
|
92
|
+
|
|
93
|
+
- [API Documentation](https://karonlabs.net/api/docs.html)
|
|
94
|
+
- [Pricing](https://karonlabs.net/api/pricing.html)
|
|
95
|
+
- [WAF Compatibility](https://karonlabs.net/api/waf.html)
|
|
96
|
+
- [Changelog](CHANGELOG.md)
|
|
97
|
+
|
|
98
|
+
## License
|
|
99
|
+
|
|
100
|
+
MIT
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "karon-mcp"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Karon Web Unblocker — MCP server for Claude, Cursor, and other MCP clients. Fetch any URL bypassing Cloudflare, DataDome, Akamai, and other WAFs."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Karon Labs", email = "ceo@karonlabs.net" },
|
|
14
|
+
]
|
|
15
|
+
keywords = ["mcp", "web-scraping", "waf-bypass", "cloudflare", "unblocker", "claude", "cursor"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Topic :: Internet :: WWW/HTTP",
|
|
26
|
+
"Topic :: Software Development :: Libraries",
|
|
27
|
+
]
|
|
28
|
+
dependencies = [
|
|
29
|
+
"mcp>=1.0.0",
|
|
30
|
+
"httpx>=0.27.0",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://karonlabs.net/api/"
|
|
35
|
+
Documentation = "https://karonlabs.net/api/docs.html"
|
|
36
|
+
Repository = "https://github.com/pplar39/karon3"
|
|
37
|
+
Changelog = "https://github.com/pplar39/karon3/blob/main/karon-mcp/CHANGELOG.md"
|
|
38
|
+
|
|
39
|
+
[project.scripts]
|
|
40
|
+
karon-mcp = "karon_mcp.__main__:main"
|
|
41
|
+
|
|
42
|
+
[tool.hatch.build.targets.wheel]
|
|
43
|
+
packages = ["src/karon_mcp"]
|
|
@@ -0,0 +1,349 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Karon Web Unblocker — MCP Server"""
|
|
3
|
+
from mcp.server.fastmcp import FastMCP
|
|
4
|
+
import httpx
|
|
5
|
+
import os
|
|
6
|
+
import json
|
|
7
|
+
import asyncio
|
|
8
|
+
import logging
|
|
9
|
+
import ipaddress
|
|
10
|
+
import socket
|
|
11
|
+
from urllib.parse import urlparse
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger("karon-unblocker")
|
|
14
|
+
|
|
15
|
+
mcp = FastMCP("karon-unblocker")
|
|
16
|
+
|
|
17
|
+
_API_BASE = "https://api.karonlabs.net"
|
|
18
|
+
_ALLOWED_EXTRACTS = {"markdown", "text", "html"}
|
|
19
|
+
_MAX_URL_LEN = 8192
|
|
20
|
+
_MAX_CONTENT_LEN = 200_000
|
|
21
|
+
_MAX_CONCURRENCY = 5
|
|
22
|
+
_MAX_URLS = 20
|
|
23
|
+
_TIMEOUT_LOCAL = 70.0
|
|
24
|
+
_MAX_RESPONSE_BYTES = 10 * 1024 * 1024
|
|
25
|
+
|
|
26
|
+
_global_sem = asyncio.Semaphore(_MAX_CONCURRENCY)
|
|
27
|
+
_client: httpx.AsyncClient | None = None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _get_api_key() -> str:
|
|
31
|
+
return os.environ.get("KARON_API_KEY", "")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
async def _get_client() -> httpx.AsyncClient:
|
|
35
|
+
global _client
|
|
36
|
+
if _client is None or _client.is_closed:
|
|
37
|
+
_client = httpx.AsyncClient(
|
|
38
|
+
timeout=_TIMEOUT_LOCAL,
|
|
39
|
+
follow_redirects=True,
|
|
40
|
+
limits=httpx.Limits(max_connections=_MAX_CONCURRENCY),
|
|
41
|
+
)
|
|
42
|
+
return _client
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
_BLOCKED_NETS = [
|
|
46
|
+
ipaddress.ip_network("0.0.0.0/8"),
|
|
47
|
+
ipaddress.ip_network("10.0.0.0/8"),
|
|
48
|
+
ipaddress.ip_network("100.64.0.0/10"),
|
|
49
|
+
ipaddress.ip_network("127.0.0.0/8"),
|
|
50
|
+
ipaddress.ip_network("169.254.0.0/16"),
|
|
51
|
+
ipaddress.ip_network("172.16.0.0/12"),
|
|
52
|
+
ipaddress.ip_network("192.0.0.0/24"),
|
|
53
|
+
ipaddress.ip_network("192.168.0.0/16"),
|
|
54
|
+
ipaddress.ip_network("::/128"),
|
|
55
|
+
ipaddress.ip_network("::1/128"),
|
|
56
|
+
ipaddress.ip_network("fc00::/7"),
|
|
57
|
+
ipaddress.ip_network("fe80::/10"),
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _is_blocked_ip(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
|
62
|
+
check_addrs = [addr]
|
|
63
|
+
if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
|
|
64
|
+
check_addrs.append(addr.ipv4_mapped)
|
|
65
|
+
for a in check_addrs:
|
|
66
|
+
for net in _BLOCKED_NETS:
|
|
67
|
+
if a in net:
|
|
68
|
+
return True
|
|
69
|
+
return False
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _validate_url(url: str) -> str | None:
|
|
73
|
+
"""Returns error message if invalid, None if OK."""
|
|
74
|
+
if not isinstance(url, str) or not url.strip():
|
|
75
|
+
return "url must be a non-empty string"
|
|
76
|
+
if len(url) > _MAX_URL_LEN:
|
|
77
|
+
return f"url exceeds {_MAX_URL_LEN} chars"
|
|
78
|
+
parsed = urlparse(url)
|
|
79
|
+
if parsed.scheme not in ("http", "https"):
|
|
80
|
+
return "url must use http or https scheme"
|
|
81
|
+
hostname = parsed.hostname
|
|
82
|
+
if not hostname:
|
|
83
|
+
return "url has no hostname"
|
|
84
|
+
hostname_lower = hostname.lower()
|
|
85
|
+
if hostname_lower in ("localhost", "localhost.localdomain", "ip6-localhost", "ip6-loopback"):
|
|
86
|
+
return "url targets a blocked loopback hostname"
|
|
87
|
+
try:
|
|
88
|
+
addr = ipaddress.ip_address(hostname_lower)
|
|
89
|
+
if _is_blocked_ip(addr):
|
|
90
|
+
return "url targets a blocked internal network"
|
|
91
|
+
except ValueError:
|
|
92
|
+
pass
|
|
93
|
+
try:
|
|
94
|
+
infos = socket.getaddrinfo(hostname, None, proto=socket.IPPROTO_TCP)
|
|
95
|
+
for info in infos:
|
|
96
|
+
resolved_ip = info[4][0]
|
|
97
|
+
try:
|
|
98
|
+
addr = ipaddress.ip_address(resolved_ip)
|
|
99
|
+
if _is_blocked_ip(addr):
|
|
100
|
+
return "url resolves to a blocked internal network"
|
|
101
|
+
except ValueError:
|
|
102
|
+
continue
|
|
103
|
+
except socket.gaierror:
|
|
104
|
+
pass
|
|
105
|
+
return None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _safe_str(val: object) -> str:
|
|
109
|
+
if isinstance(val, str):
|
|
110
|
+
return val.translate({i: f"\\x{i:02x}" for i in range(32) if i not in (10, 13)})
|
|
111
|
+
return str(val)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _parse_response(data: object) -> dict | None:
|
|
115
|
+
"""Validate upstream response is a dict. Returns None if not."""
|
|
116
|
+
if not isinstance(data, dict):
|
|
117
|
+
return None
|
|
118
|
+
return data
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _build_error(msg: str, status_code: int | None = None, cost: object = None) -> str:
|
|
122
|
+
parts = [f"Error: {msg}"]
|
|
123
|
+
if status_code is not None:
|
|
124
|
+
parts.append(f"(HTTP {status_code})")
|
|
125
|
+
if cost is not None:
|
|
126
|
+
parts.append(f"[credits_used: {cost}]")
|
|
127
|
+
return " ".join(parts)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@mcp.tool()
|
|
131
|
+
async def browse(
|
|
132
|
+
url: str,
|
|
133
|
+
extract: str = "markdown",
|
|
134
|
+
readability: bool = True,
|
|
135
|
+
) -> str:
|
|
136
|
+
"""
|
|
137
|
+
Fetch any URL bypassing WAFs (Cloudflare, DataDome, Kasada, Akamai, PerimeterX).
|
|
138
|
+
Returns clean text or markdown. Costs 1 credit (cache) or 10 credits (browser).
|
|
139
|
+
|
|
140
|
+
Args:
|
|
141
|
+
url: Target URL (must start with http/https)
|
|
142
|
+
extract: Output format — "markdown" | "text" | "html"
|
|
143
|
+
readability: True = main content only (default). False = full page.
|
|
144
|
+
"""
|
|
145
|
+
api_key = _get_api_key()
|
|
146
|
+
if not api_key:
|
|
147
|
+
return _build_error("KARON_API_KEY environment variable not set")
|
|
148
|
+
|
|
149
|
+
url_err = _validate_url(url)
|
|
150
|
+
if url_err:
|
|
151
|
+
return _build_error(url_err)
|
|
152
|
+
|
|
153
|
+
if extract not in _ALLOWED_EXTRACTS:
|
|
154
|
+
return _build_error(f"extract must be one of {_ALLOWED_EXTRACTS}")
|
|
155
|
+
|
|
156
|
+
if not isinstance(readability, bool):
|
|
157
|
+
return _build_error("readability must be a boolean")
|
|
158
|
+
|
|
159
|
+
async with _global_sem:
|
|
160
|
+
try:
|
|
161
|
+
client = await _get_client()
|
|
162
|
+
r = await client.post(
|
|
163
|
+
f"{_API_BASE}/v1/agent/browse",
|
|
164
|
+
headers={"Authorization": f"Bearer {api_key}"},
|
|
165
|
+
json={"url": url, "extract": extract, "readability": readability},
|
|
166
|
+
)
|
|
167
|
+
except Exception as e:
|
|
168
|
+
logger.error("browse request failed", exc_info=True)
|
|
169
|
+
return _build_error(str(e))
|
|
170
|
+
|
|
171
|
+
status_code = r.status_code
|
|
172
|
+
try:
|
|
173
|
+
raw = r.content
|
|
174
|
+
if len(raw) > _MAX_RESPONSE_BYTES:
|
|
175
|
+
return _build_error("response too large", status_code=status_code)
|
|
176
|
+
data = json.loads(raw)
|
|
177
|
+
except Exception as e:
|
|
178
|
+
logger.error("browse JSON parse failed (HTTP %d)", status_code, exc_info=True)
|
|
179
|
+
return _build_error(f"JSON parse failed: {e}", status_code=status_code)
|
|
180
|
+
|
|
181
|
+
data = _parse_response(data)
|
|
182
|
+
if data is None:
|
|
183
|
+
return _build_error("unexpected response format", status_code=status_code)
|
|
184
|
+
|
|
185
|
+
success = data.get("success")
|
|
186
|
+
cost = data.get("cost_credits")
|
|
187
|
+
|
|
188
|
+
if success is not True:
|
|
189
|
+
return _build_error(
|
|
190
|
+
data.get("error", "unknown"),
|
|
191
|
+
status_code=status_code,
|
|
192
|
+
cost=cost,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
if status_code >= 400:
|
|
196
|
+
return _build_error("HTTP error with success flag", status_code=status_code, cost=cost)
|
|
197
|
+
|
|
198
|
+
content = _safe_str(data.get("content", ""))
|
|
199
|
+
if len(content) > _MAX_CONTENT_LEN:
|
|
200
|
+
content = content[:_MAX_CONTENT_LEN] + "\n[truncated]"
|
|
201
|
+
|
|
202
|
+
resolved_url = _safe_str(data.get("url", url))
|
|
203
|
+
timing = data.get("timing")
|
|
204
|
+
cache_hit = timing.get("cache_hit", "?") if isinstance(timing, dict) else "?"
|
|
205
|
+
|
|
206
|
+
if extract == "html":
|
|
207
|
+
return json.dumps({
|
|
208
|
+
"source": resolved_url,
|
|
209
|
+
"credits_used": cost,
|
|
210
|
+
"cache_hit": cache_hit,
|
|
211
|
+
"content": content,
|
|
212
|
+
}, ensure_ascii=False)
|
|
213
|
+
|
|
214
|
+
lines = [
|
|
215
|
+
f"[source]: {resolved_url}",
|
|
216
|
+
f"[credits_used]: {cost if cost is not None else '?'}",
|
|
217
|
+
f"[cache_hit]: {cache_hit}",
|
|
218
|
+
"---",
|
|
219
|
+
content,
|
|
220
|
+
]
|
|
221
|
+
return "\n".join(lines)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
@mcp.tool()
|
|
225
|
+
async def crawl(
|
|
226
|
+
urls: list[str],
|
|
227
|
+
extract: str = "markdown",
|
|
228
|
+
readability: bool = True,
|
|
229
|
+
concurrency: int = 3,
|
|
230
|
+
) -> str:
|
|
231
|
+
"""
|
|
232
|
+
Fetch multiple URLs concurrently bypassing WAFs. Returns JSON array of results.
|
|
233
|
+
|
|
234
|
+
Args:
|
|
235
|
+
urls: List of URLs (max 20)
|
|
236
|
+
extract: Output format — "markdown" | "text"
|
|
237
|
+
readability: True = main content only (default). False = full page.
|
|
238
|
+
concurrency: Parallel requests (1-5, default 3)
|
|
239
|
+
"""
|
|
240
|
+
api_key = _get_api_key()
|
|
241
|
+
if not api_key:
|
|
242
|
+
return json.dumps([{"url": "N/A", "success": False, "error": "KARON_API_KEY environment variable not set"}])
|
|
243
|
+
|
|
244
|
+
if not isinstance(urls, list):
|
|
245
|
+
return json.dumps([{"url": "N/A", "success": False, "error": "urls must be a list of strings"}])
|
|
246
|
+
|
|
247
|
+
if not isinstance(concurrency, int):
|
|
248
|
+
return json.dumps([{"url": "N/A", "success": False, "error": "concurrency must be an integer"}])
|
|
249
|
+
|
|
250
|
+
if extract not in _ALLOWED_EXTRACTS:
|
|
251
|
+
return json.dumps([{"url": "N/A", "success": False, "error": f"extract must be one of {sorted(_ALLOWED_EXTRACTS)}"}])
|
|
252
|
+
|
|
253
|
+
if not isinstance(readability, bool):
|
|
254
|
+
return json.dumps([{"url": "N/A", "success": False, "error": "readability must be a boolean"}])
|
|
255
|
+
|
|
256
|
+
concurrency = max(1, min(_MAX_CONCURRENCY, concurrency))
|
|
257
|
+
|
|
258
|
+
validated: list[str] = []
|
|
259
|
+
rejected: list[dict] = []
|
|
260
|
+
seen: set[str] = set()
|
|
261
|
+
for u in urls:
|
|
262
|
+
if not isinstance(u, str) or not u.strip():
|
|
263
|
+
rejected.append({"url": str(u), "error": "invalid url type or empty"})
|
|
264
|
+
continue
|
|
265
|
+
url_err = _validate_url(u)
|
|
266
|
+
if url_err:
|
|
267
|
+
rejected.append({"url": u, "error": url_err})
|
|
268
|
+
continue
|
|
269
|
+
if u in seen:
|
|
270
|
+
continue
|
|
271
|
+
seen.add(u)
|
|
272
|
+
validated.append(u)
|
|
273
|
+
|
|
274
|
+
if len(validated) > _MAX_URLS:
|
|
275
|
+
return json.dumps([{
|
|
276
|
+
"url": "N/A", "success": False,
|
|
277
|
+
"error": f"too many URLs: {len(validated)} (max {_MAX_URLS})",
|
|
278
|
+
}])
|
|
279
|
+
|
|
280
|
+
local_sem = asyncio.Semaphore(concurrency)
|
|
281
|
+
|
|
282
|
+
async def fetch_one(target_url: str) -> dict:
|
|
283
|
+
async with _global_sem:
|
|
284
|
+
async with local_sem:
|
|
285
|
+
try:
|
|
286
|
+
client = await _get_client()
|
|
287
|
+
r = await client.post(
|
|
288
|
+
f"{_API_BASE}/v1/agent/browse",
|
|
289
|
+
headers={"Authorization": f"Bearer {api_key}"},
|
|
290
|
+
json={"url": target_url, "extract": extract, "readability": readability},
|
|
291
|
+
)
|
|
292
|
+
except Exception as e:
|
|
293
|
+
logger.error("crawl fetch failed for %s", target_url, exc_info=True)
|
|
294
|
+
return {"url": target_url, "success": False, "content": "", "error": str(e)}
|
|
295
|
+
|
|
296
|
+
status_code = r.status_code
|
|
297
|
+
try:
|
|
298
|
+
raw = r.content
|
|
299
|
+
if len(raw) > _MAX_RESPONSE_BYTES:
|
|
300
|
+
return {"url": target_url, "success": False, "content": "", "error": "response too large", "status_code": status_code}
|
|
301
|
+
data = json.loads(raw)
|
|
302
|
+
except Exception as e:
|
|
303
|
+
logger.error("crawl JSON parse failed for %s (HTTP %d)", target_url, status_code, exc_info=True)
|
|
304
|
+
return {"url": target_url, "success": False, "content": "", "error": f"JSON parse: {e}", "status_code": status_code}
|
|
305
|
+
|
|
306
|
+
if not isinstance(data, dict):
|
|
307
|
+
return {"url": target_url, "success": False, "content": "", "error": "unexpected response format", "status_code": status_code}
|
|
308
|
+
|
|
309
|
+
success = data.get("success") is True
|
|
310
|
+
content = _safe_str(data.get("content", ""))
|
|
311
|
+
if len(content) > _MAX_CONTENT_LEN:
|
|
312
|
+
content = content[:_MAX_CONTENT_LEN] + "\n[truncated]"
|
|
313
|
+
|
|
314
|
+
return {
|
|
315
|
+
"url": target_url,
|
|
316
|
+
"resolved_url": data.get("url", target_url),
|
|
317
|
+
"success": success,
|
|
318
|
+
"content": content,
|
|
319
|
+
"error": data.get("error"),
|
|
320
|
+
"cost_credits": data.get("cost_credits"),
|
|
321
|
+
"status_code": status_code,
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
tasks = [fetch_one(u) for u in validated]
|
|
325
|
+
try:
|
|
326
|
+
results = await asyncio.wait_for(
|
|
327
|
+
asyncio.gather(*tasks, return_exceptions=True),
|
|
328
|
+
timeout=_TIMEOUT_LOCAL * len(validated) / max(concurrency, 1) + 30,
|
|
329
|
+
)
|
|
330
|
+
except asyncio.TimeoutError:
|
|
331
|
+
results = [{"url": "batch", "success": False, "error": "batch timeout exceeded"}]
|
|
332
|
+
|
|
333
|
+
final: list[dict] = []
|
|
334
|
+
for r in results:
|
|
335
|
+
if isinstance(r, BaseException):
|
|
336
|
+
final.append({"url": "?", "success": False, "error": str(r)})
|
|
337
|
+
else:
|
|
338
|
+
final.append(r)
|
|
339
|
+
|
|
340
|
+
if rejected:
|
|
341
|
+
for rej in rejected:
|
|
342
|
+
final.append({"url": rej["url"], "success": False, "content": "", "error": rej["error"]})
|
|
343
|
+
|
|
344
|
+
return json.dumps(final, ensure_ascii=False, indent=2, allow_nan=False)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
if __name__ == "__main__":
|
|
348
|
+
from karon_mcp.__main__ import main
|
|
349
|
+
main()
|