karon-mcp 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,42 @@
1
+ # Secrets — NEVER commit
2
+ .env
3
+ *.pem
4
+ *.key
5
+
6
+ # Build artifacts
7
+ /target/
8
+
9
+ # Logs
10
+ bot.log
11
+ build.log
12
+ *.log
13
+
14
+ # Backups
15
+ *.bak
16
+ *.bak_
17
+ /backups/
18
+
19
+ # OS / IDE
20
+ .DS_Store
21
+ .vscode/
22
+ .idea/
23
+ *.swp
24
+ *.swo
25
+
26
+ # Misc
27
+ /tmp/
28
+ target/
29
+
30
+ # OpenClaw runtime
31
+ openclaw-stack/node_modules/
32
+ openclaw-stack/profiles/
33
+ openclaw-stack/blitz_logs/
34
+ openclaw-stack/doctor/
35
+ openclaw-stack/package-lock.json
36
+
37
+ # Sensitive
38
+ .ssh/
39
+ wallet.json
40
+ ssh/
41
+ .env.bak*
42
+ *oauth*.json
@@ -0,0 +1,15 @@
1
+ # Changelog
2
+
3
+ All notable changes to `karon-mcp` will be documented in this file.
4
+
5
+ ## [1.0.0] - 2026-04-08
6
+
7
+ ### Added
8
+ - `browse` tool — Fetch any URL bypassing WAFs (Cloudflare, DataDome, Kasada, Akamai, PerimeterX). Returns markdown, text, or raw HTML.
9
+ - `crawl` tool — Fetch up to 20 URLs concurrently with configurable parallelism (1-5).
10
+ - SSRF protection — Blocks requests to internal networks (RFC 1918, loopback, link-local, IPv4-mapped IPv6).
11
+ - DNS rebinding protection — Resolves hostnames and validates against blocked ranges.
12
+ - Response size limits — 10 MB max response, 200K content truncation.
13
+ - Global concurrency semaphore — Max 5 concurrent requests to protect upstream API.
14
+ - Singleton HTTP client with connection pooling.
15
+ - Control character escaping in all output strings.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Karon Labs
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,127 @@
1
+ Metadata-Version: 2.4
2
+ Name: karon-mcp
3
+ Version: 1.0.0
4
+ Summary: Karon Web Unblocker — MCP server for Claude, Cursor, and other MCP clients. Fetch any URL bypassing Cloudflare, DataDome, Akamai, and other WAFs.
5
+ Project-URL: Homepage, https://karonlabs.net/api/
6
+ Project-URL: Documentation, https://karonlabs.net/api/docs.html
7
+ Project-URL: Repository, https://github.com/pplar39/karon3
8
+ Project-URL: Changelog, https://github.com/pplar39/karon3/blob/main/karon-mcp/CHANGELOG.md
9
+ Author-email: Karon Labs <ceo@karonlabs.net>
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: claude,cloudflare,cursor,mcp,unblocker,waf-bypass,web-scraping
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Topic :: Internet :: WWW/HTTP
22
+ Classifier: Topic :: Software Development :: Libraries
23
+ Requires-Python: >=3.10
24
+ Requires-Dist: httpx>=0.27.0
25
+ Requires-Dist: mcp>=1.0.0
26
+ Description-Content-Type: text/markdown
27
+
28
+ # karon-mcp
29
+
30
+ MCP server for the [Karon Web Unblocker API](https://karonlabs.net/api/). Fetch any URL bypassing Cloudflare, DataDome, Akamai, Kasada, and PerimeterX — directly from Claude, Cursor, or any MCP-compatible client.
31
+
32
+ ## Quick Start
33
+
34
+ ### 1. Get your API key
35
+
36
+ Sign up at [karonlabs.net/api/signup.html](https://karonlabs.net/api/signup.html) (Google login, free tier: 1,000 credits).
37
+
38
+ ### 2. Install
39
+
40
+ **Claude Desktop / Claude Code** — add to your MCP config (`claude_desktop_config.json` or `.mcp.json`):
41
+
42
+ ```json
43
+ {
44
+ "mcpServers": {
45
+ "karon": {
46
+ "command": "uvx",
47
+ "args": ["karon-mcp"],
48
+ "env": {
49
+ "KARON_API_KEY": "your_api_key_here"
50
+ }
51
+ }
52
+ }
53
+ }
54
+ ```
55
+
56
+ > `uvx` auto-fetches the latest version from PyPI on every run. No manual updates needed.
57
+
58
+ **Cursor** — go to Settings > MCP Servers > Add, then use the same config above.
59
+
60
+ **Manual install** (if you prefer pip):
61
+
62
+ ```bash
63
+ pip install karon-mcp
64
+ ```
65
+
66
+ ### 3. Use
67
+
68
+ Once installed, two tools become available in your MCP client:
69
+
70
+ #### `browse` — Fetch a single URL
71
+
72
+ ```
73
+ browse(url="https://example.com", extract="markdown", readability=True)
74
+ ```
75
+
76
+ | Parameter | Type | Default | Description |
77
+ |-----------|------|---------|-------------|
78
+ | `url` | string | required | Target URL (http/https) |
79
+ | `extract` | string | `"markdown"` | Output format: `"markdown"`, `"text"`, or `"html"` |
80
+ | `readability` | bool | `True` | `True` = main content only, `False` = full page |
81
+
82
+ #### `crawl` — Fetch multiple URLs concurrently
83
+
84
+ ```
85
+ crawl(urls=["https://a.com", "https://b.com"], extract="markdown", concurrency=3)
86
+ ```
87
+
88
+ | Parameter | Type | Default | Description |
89
+ |-----------|------|---------|-------------|
90
+ | `urls` | list[string] | required | Up to 20 URLs |
91
+ | `extract` | string | `"markdown"` | Output format: `"markdown"` or `"text"` |
92
+ | `readability` | bool | `True` | Main content only |
93
+ | `concurrency` | int | `3` | Parallel requests (1-5) |
94
+
95
+ ### Credits
96
+
97
+ | Method | Cost |
98
+ |--------|------|
99
+ | Cache hit | 1 credit |
100
+ | Browser render | 10 credits |
101
+
102
+ Free tier includes 1,000 credits. See [pricing](https://karonlabs.net/api/pricing.html) for paid plans.
103
+
104
+ ## Configuration
105
+
106
+ The only required configuration is the `KARON_API_KEY` environment variable. Get yours at [karonlabs.net/api/signup.html](https://karonlabs.net/api/signup.html).
107
+
108
+ ## Updating
109
+
110
+ If you're using `uvx` (recommended), updates are automatic — every run fetches the latest version.
111
+
112
+ If you installed with pip:
113
+
114
+ ```bash
115
+ pip install --upgrade karon-mcp
116
+ ```
117
+
118
+ ## Links
119
+
120
+ - [API Documentation](https://karonlabs.net/api/docs.html)
121
+ - [Pricing](https://karonlabs.net/api/pricing.html)
122
+ - [WAF Compatibility](https://karonlabs.net/api/waf.html)
123
+ - [Changelog](CHANGELOG.md)
124
+
125
+ ## License
126
+
127
+ MIT
@@ -0,0 +1,100 @@
1
+ # karon-mcp
2
+
3
+ MCP server for the [Karon Web Unblocker API](https://karonlabs.net/api/). Fetch any URL bypassing Cloudflare, DataDome, Akamai, Kasada, and PerimeterX — directly from Claude, Cursor, or any MCP-compatible client.
4
+
5
+ ## Quick Start
6
+
7
+ ### 1. Get your API key
8
+
9
+ Sign up at [karonlabs.net/api/signup.html](https://karonlabs.net/api/signup.html) (Google login, free tier: 1,000 credits).
10
+
11
+ ### 2. Install
12
+
13
+ **Claude Desktop / Claude Code** — add to your MCP config (`claude_desktop_config.json` or `.mcp.json`):
14
+
15
+ ```json
16
+ {
17
+ "mcpServers": {
18
+ "karon": {
19
+ "command": "uvx",
20
+ "args": ["karon-mcp"],
21
+ "env": {
22
+ "KARON_API_KEY": "your_api_key_here"
23
+ }
24
+ }
25
+ }
26
+ }
27
+ ```
28
+
29
+ > `uvx` auto-fetches the latest version from PyPI on every run. No manual updates needed.
30
+
31
+ **Cursor** — go to Settings > MCP Servers > Add, then use the same config above.
32
+
33
+ **Manual install** (if you prefer pip):
34
+
35
+ ```bash
36
+ pip install karon-mcp
37
+ ```
38
+
39
+ ### 3. Use
40
+
41
+ Once installed, two tools become available in your MCP client:
42
+
43
+ #### `browse` — Fetch a single URL
44
+
45
+ ```
46
+ browse(url="https://example.com", extract="markdown", readability=True)
47
+ ```
48
+
49
+ | Parameter | Type | Default | Description |
50
+ |-----------|------|---------|-------------|
51
+ | `url` | string | required | Target URL (http/https) |
52
+ | `extract` | string | `"markdown"` | Output format: `"markdown"`, `"text"`, or `"html"` |
53
+ | `readability` | bool | `True` | `True` = main content only, `False` = full page |
54
+
55
+ #### `crawl` — Fetch multiple URLs concurrently
56
+
57
+ ```
58
+ crawl(urls=["https://a.com", "https://b.com"], extract="markdown", concurrency=3)
59
+ ```
60
+
61
+ | Parameter | Type | Default | Description |
62
+ |-----------|------|---------|-------------|
63
+ | `urls` | list[string] | required | Up to 20 URLs |
64
+ | `extract` | string | `"markdown"` | Output format: `"markdown"` or `"text"` |
65
+ | `readability` | bool | `True` | Main content only |
66
+ | `concurrency` | int | `3` | Parallel requests (1-5) |
67
+
68
+ ### Credits
69
+
70
+ | Method | Cost |
71
+ |--------|------|
72
+ | Cache hit | 1 credit |
73
+ | Browser render | 10 credits |
74
+
75
+ Free tier includes 1,000 credits. See [pricing](https://karonlabs.net/api/pricing.html) for paid plans.
76
+
77
+ ## Configuration
78
+
79
+ The only required configuration is the `KARON_API_KEY` environment variable. Get yours at [karonlabs.net/api/signup.html](https://karonlabs.net/api/signup.html).
80
+
81
+ ## Updating
82
+
83
+ If you're using `uvx` (recommended), updates are automatic — every run fetches the latest version.
84
+
85
+ If you installed with pip:
86
+
87
+ ```bash
88
+ pip install --upgrade karon-mcp
89
+ ```
90
+
91
+ ## Links
92
+
93
+ - [API Documentation](https://karonlabs.net/api/docs.html)
94
+ - [Pricing](https://karonlabs.net/api/pricing.html)
95
+ - [WAF Compatibility](https://karonlabs.net/api/waf.html)
96
+ - [Changelog](CHANGELOG.md)
97
+
98
+ ## License
99
+
100
+ MIT
@@ -0,0 +1,43 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "karon-mcp"
7
+ version = "1.0.0"
8
+ description = "Karon Web Unblocker — MCP server for Claude, Cursor, and other MCP clients. Fetch any URL bypassing Cloudflare, DataDome, Akamai, and other WAFs."
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ requires-python = ">=3.10"
12
+ authors = [
13
+ { name = "Karon Labs", email = "ceo@karonlabs.net" },
14
+ ]
15
+ keywords = ["mcp", "web-scraping", "waf-bypass", "cloudflare", "unblocker", "claude", "cursor"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Intended Audience :: Developers",
19
+ "License :: OSI Approved :: MIT License",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: Python :: 3.10",
22
+ "Programming Language :: Python :: 3.11",
23
+ "Programming Language :: Python :: 3.12",
24
+ "Programming Language :: Python :: 3.13",
25
+ "Topic :: Internet :: WWW/HTTP",
26
+ "Topic :: Software Development :: Libraries",
27
+ ]
28
+ dependencies = [
29
+ "mcp>=1.0.0",
30
+ "httpx>=0.27.0",
31
+ ]
32
+
33
+ [project.urls]
34
+ Homepage = "https://karonlabs.net/api/"
35
+ Documentation = "https://karonlabs.net/api/docs.html"
36
+ Repository = "https://github.com/pplar39/karon3"
37
+ Changelog = "https://github.com/pplar39/karon3/blob/main/karon-mcp/CHANGELOG.md"
38
+
39
+ [project.scripts]
40
+ karon-mcp = "karon_mcp.__main__:main"
41
+
42
+ [tool.hatch.build.targets.wheel]
43
+ packages = ["src/karon_mcp"]
@@ -0,0 +1,3 @@
1
+ """Karon Web Unblocker — MCP Server for Claude, Cursor, and other MCP clients."""
2
+
3
+ __version__ = "1.0.0"
@@ -0,0 +1,10 @@
1
+ """Entry point for `python -m karon_mcp` and `uvx karon-mcp`."""
2
+ from karon_mcp.server import mcp
3
+
4
+
5
+ def main():
6
+ mcp.run()
7
+
8
+
9
+ if __name__ == "__main__":
10
+ main()
@@ -0,0 +1,349 @@
1
+ #!/usr/bin/env python3
2
+ """Karon Web Unblocker — MCP Server"""
3
+ from mcp.server.fastmcp import FastMCP
4
+ import httpx
5
+ import os
6
+ import json
7
+ import asyncio
8
+ import logging
9
+ import ipaddress
10
+ import socket
11
+ from urllib.parse import urlparse
12
+
13
+ logger = logging.getLogger("karon-unblocker")
14
+
15
+ mcp = FastMCP("karon-unblocker")
16
+
17
+ _API_BASE = "https://api.karonlabs.net"
18
+ _ALLOWED_EXTRACTS = {"markdown", "text", "html"}
19
+ _MAX_URL_LEN = 8192
20
+ _MAX_CONTENT_LEN = 200_000
21
+ _MAX_CONCURRENCY = 5
22
+ _MAX_URLS = 20
23
+ _TIMEOUT_LOCAL = 70.0
24
+ _MAX_RESPONSE_BYTES = 10 * 1024 * 1024
25
+
26
+ _global_sem = asyncio.Semaphore(_MAX_CONCURRENCY)
27
+ _client: httpx.AsyncClient | None = None
28
+
29
+
30
+ def _get_api_key() -> str:
31
+ return os.environ.get("KARON_API_KEY", "")
32
+
33
+
34
+ async def _get_client() -> httpx.AsyncClient:
35
+ global _client
36
+ if _client is None or _client.is_closed:
37
+ _client = httpx.AsyncClient(
38
+ timeout=_TIMEOUT_LOCAL,
39
+ follow_redirects=True,
40
+ limits=httpx.Limits(max_connections=_MAX_CONCURRENCY),
41
+ )
42
+ return _client
43
+
44
+
45
+ _BLOCKED_NETS = [
46
+ ipaddress.ip_network("0.0.0.0/8"),
47
+ ipaddress.ip_network("10.0.0.0/8"),
48
+ ipaddress.ip_network("100.64.0.0/10"),
49
+ ipaddress.ip_network("127.0.0.0/8"),
50
+ ipaddress.ip_network("169.254.0.0/16"),
51
+ ipaddress.ip_network("172.16.0.0/12"),
52
+ ipaddress.ip_network("192.0.0.0/24"),
53
+ ipaddress.ip_network("192.168.0.0/16"),
54
+ ipaddress.ip_network("::/128"),
55
+ ipaddress.ip_network("::1/128"),
56
+ ipaddress.ip_network("fc00::/7"),
57
+ ipaddress.ip_network("fe80::/10"),
58
+ ]
59
+
60
+
61
+ def _is_blocked_ip(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
62
+ check_addrs = [addr]
63
+ if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
64
+ check_addrs.append(addr.ipv4_mapped)
65
+ for a in check_addrs:
66
+ for net in _BLOCKED_NETS:
67
+ if a in net:
68
+ return True
69
+ return False
70
+
71
+
72
+ def _validate_url(url: str) -> str | None:
73
+ """Returns error message if invalid, None if OK."""
74
+ if not isinstance(url, str) or not url.strip():
75
+ return "url must be a non-empty string"
76
+ if len(url) > _MAX_URL_LEN:
77
+ return f"url exceeds {_MAX_URL_LEN} chars"
78
+ parsed = urlparse(url)
79
+ if parsed.scheme not in ("http", "https"):
80
+ return "url must use http or https scheme"
81
+ hostname = parsed.hostname
82
+ if not hostname:
83
+ return "url has no hostname"
84
+ hostname_lower = hostname.lower()
85
+ if hostname_lower in ("localhost", "localhost.localdomain", "ip6-localhost", "ip6-loopback"):
86
+ return "url targets a blocked loopback hostname"
87
+ try:
88
+ addr = ipaddress.ip_address(hostname_lower)
89
+ if _is_blocked_ip(addr):
90
+ return "url targets a blocked internal network"
91
+ except ValueError:
92
+ pass
93
+ try:
94
+ infos = socket.getaddrinfo(hostname, None, proto=socket.IPPROTO_TCP)
95
+ for info in infos:
96
+ resolved_ip = info[4][0]
97
+ try:
98
+ addr = ipaddress.ip_address(resolved_ip)
99
+ if _is_blocked_ip(addr):
100
+ return "url resolves to a blocked internal network"
101
+ except ValueError:
102
+ continue
103
+ except socket.gaierror:
104
+ pass
105
+ return None
106
+
107
+
108
+ def _safe_str(val: object) -> str:
109
+ if isinstance(val, str):
110
+ return val.translate({i: f"\\x{i:02x}" for i in range(32) if i not in (10, 13)})
111
+ return str(val)
112
+
113
+
114
+ def _parse_response(data: object) -> dict | None:
115
+ """Validate upstream response is a dict. Returns None if not."""
116
+ if not isinstance(data, dict):
117
+ return None
118
+ return data
119
+
120
+
121
+ def _build_error(msg: str, status_code: int | None = None, cost: object = None) -> str:
122
+ parts = [f"Error: {msg}"]
123
+ if status_code is not None:
124
+ parts.append(f"(HTTP {status_code})")
125
+ if cost is not None:
126
+ parts.append(f"[credits_used: {cost}]")
127
+ return " ".join(parts)
128
+
129
+
130
+ @mcp.tool()
131
+ async def browse(
132
+ url: str,
133
+ extract: str = "markdown",
134
+ readability: bool = True,
135
+ ) -> str:
136
+ """
137
+ Fetch any URL bypassing WAFs (Cloudflare, DataDome, Kasada, Akamai, PerimeterX).
138
+ Returns clean text or markdown. Costs 1 credit (cache) or 10 credits (browser).
139
+
140
+ Args:
141
+ url: Target URL (must start with http/https)
142
+ extract: Output format — "markdown" | "text" | "html"
143
+ readability: True = main content only (default). False = full page.
144
+ """
145
+ api_key = _get_api_key()
146
+ if not api_key:
147
+ return _build_error("KARON_API_KEY environment variable not set")
148
+
149
+ url_err = _validate_url(url)
150
+ if url_err:
151
+ return _build_error(url_err)
152
+
153
+ if extract not in _ALLOWED_EXTRACTS:
154
+ return _build_error(f"extract must be one of {_ALLOWED_EXTRACTS}")
155
+
156
+ if not isinstance(readability, bool):
157
+ return _build_error("readability must be a boolean")
158
+
159
+ async with _global_sem:
160
+ try:
161
+ client = await _get_client()
162
+ r = await client.post(
163
+ f"{_API_BASE}/v1/agent/browse",
164
+ headers={"Authorization": f"Bearer {api_key}"},
165
+ json={"url": url, "extract": extract, "readability": readability},
166
+ )
167
+ except Exception as e:
168
+ logger.error("browse request failed", exc_info=True)
169
+ return _build_error(str(e))
170
+
171
+ status_code = r.status_code
172
+ try:
173
+ raw = r.content
174
+ if len(raw) > _MAX_RESPONSE_BYTES:
175
+ return _build_error("response too large", status_code=status_code)
176
+ data = json.loads(raw)
177
+ except Exception as e:
178
+ logger.error("browse JSON parse failed (HTTP %d)", status_code, exc_info=True)
179
+ return _build_error(f"JSON parse failed: {e}", status_code=status_code)
180
+
181
+ data = _parse_response(data)
182
+ if data is None:
183
+ return _build_error("unexpected response format", status_code=status_code)
184
+
185
+ success = data.get("success")
186
+ cost = data.get("cost_credits")
187
+
188
+ if success is not True:
189
+ return _build_error(
190
+ data.get("error", "unknown"),
191
+ status_code=status_code,
192
+ cost=cost,
193
+ )
194
+
195
+ if status_code >= 400:
196
+ return _build_error("HTTP error with success flag", status_code=status_code, cost=cost)
197
+
198
+ content = _safe_str(data.get("content", ""))
199
+ if len(content) > _MAX_CONTENT_LEN:
200
+ content = content[:_MAX_CONTENT_LEN] + "\n[truncated]"
201
+
202
+ resolved_url = _safe_str(data.get("url", url))
203
+ timing = data.get("timing")
204
+ cache_hit = timing.get("cache_hit", "?") if isinstance(timing, dict) else "?"
205
+
206
+ if extract == "html":
207
+ return json.dumps({
208
+ "source": resolved_url,
209
+ "credits_used": cost,
210
+ "cache_hit": cache_hit,
211
+ "content": content,
212
+ }, ensure_ascii=False)
213
+
214
+ lines = [
215
+ f"[source]: {resolved_url}",
216
+ f"[credits_used]: {cost if cost is not None else '?'}",
217
+ f"[cache_hit]: {cache_hit}",
218
+ "---",
219
+ content,
220
+ ]
221
+ return "\n".join(lines)
222
+
223
+
224
+ @mcp.tool()
225
+ async def crawl(
226
+ urls: list[str],
227
+ extract: str = "markdown",
228
+ readability: bool = True,
229
+ concurrency: int = 3,
230
+ ) -> str:
231
+ """
232
+ Fetch multiple URLs concurrently bypassing WAFs. Returns JSON array of results.
233
+
234
+ Args:
235
+ urls: List of URLs (max 20)
236
+ extract: Output format — "markdown" | "text"
237
+ readability: True = main content only (default). False = full page.
238
+ concurrency: Parallel requests (1-5, default 3)
239
+ """
240
+ api_key = _get_api_key()
241
+ if not api_key:
242
+ return json.dumps([{"url": "N/A", "success": False, "error": "KARON_API_KEY environment variable not set"}])
243
+
244
+ if not isinstance(urls, list):
245
+ return json.dumps([{"url": "N/A", "success": False, "error": "urls must be a list of strings"}])
246
+
247
+ if not isinstance(concurrency, int):
248
+ return json.dumps([{"url": "N/A", "success": False, "error": "concurrency must be an integer"}])
249
+
250
+ if extract not in _ALLOWED_EXTRACTS:
251
+ return json.dumps([{"url": "N/A", "success": False, "error": f"extract must be one of {sorted(_ALLOWED_EXTRACTS)}"}])
252
+
253
+ if not isinstance(readability, bool):
254
+ return json.dumps([{"url": "N/A", "success": False, "error": "readability must be a boolean"}])
255
+
256
+ concurrency = max(1, min(_MAX_CONCURRENCY, concurrency))
257
+
258
+ validated: list[str] = []
259
+ rejected: list[dict] = []
260
+ seen: set[str] = set()
261
+ for u in urls:
262
+ if not isinstance(u, str) or not u.strip():
263
+ rejected.append({"url": str(u), "error": "invalid url type or empty"})
264
+ continue
265
+ url_err = _validate_url(u)
266
+ if url_err:
267
+ rejected.append({"url": u, "error": url_err})
268
+ continue
269
+ if u in seen:
270
+ continue
271
+ seen.add(u)
272
+ validated.append(u)
273
+
274
+ if len(validated) > _MAX_URLS:
275
+ return json.dumps([{
276
+ "url": "N/A", "success": False,
277
+ "error": f"too many URLs: {len(validated)} (max {_MAX_URLS})",
278
+ }])
279
+
280
+ local_sem = asyncio.Semaphore(concurrency)
281
+
282
+ async def fetch_one(target_url: str) -> dict:
283
+ async with _global_sem:
284
+ async with local_sem:
285
+ try:
286
+ client = await _get_client()
287
+ r = await client.post(
288
+ f"{_API_BASE}/v1/agent/browse",
289
+ headers={"Authorization": f"Bearer {api_key}"},
290
+ json={"url": target_url, "extract": extract, "readability": readability},
291
+ )
292
+ except Exception as e:
293
+ logger.error("crawl fetch failed for %s", target_url, exc_info=True)
294
+ return {"url": target_url, "success": False, "content": "", "error": str(e)}
295
+
296
+ status_code = r.status_code
297
+ try:
298
+ raw = r.content
299
+ if len(raw) > _MAX_RESPONSE_BYTES:
300
+ return {"url": target_url, "success": False, "content": "", "error": "response too large", "status_code": status_code}
301
+ data = json.loads(raw)
302
+ except Exception as e:
303
+ logger.error("crawl JSON parse failed for %s (HTTP %d)", target_url, status_code, exc_info=True)
304
+ return {"url": target_url, "success": False, "content": "", "error": f"JSON parse: {e}", "status_code": status_code}
305
+
306
+ if not isinstance(data, dict):
307
+ return {"url": target_url, "success": False, "content": "", "error": "unexpected response format", "status_code": status_code}
308
+
309
+ success = data.get("success") is True
310
+ content = _safe_str(data.get("content", ""))
311
+ if len(content) > _MAX_CONTENT_LEN:
312
+ content = content[:_MAX_CONTENT_LEN] + "\n[truncated]"
313
+
314
+ return {
315
+ "url": target_url,
316
+ "resolved_url": data.get("url", target_url),
317
+ "success": success,
318
+ "content": content,
319
+ "error": data.get("error"),
320
+ "cost_credits": data.get("cost_credits"),
321
+ "status_code": status_code,
322
+ }
323
+
324
+ tasks = [fetch_one(u) for u in validated]
325
+ try:
326
+ results = await asyncio.wait_for(
327
+ asyncio.gather(*tasks, return_exceptions=True),
328
+ timeout=_TIMEOUT_LOCAL * len(validated) / max(concurrency, 1) + 30,
329
+ )
330
+ except asyncio.TimeoutError:
331
+ results = [{"url": "batch", "success": False, "error": "batch timeout exceeded"}]
332
+
333
+ final: list[dict] = []
334
+ for r in results:
335
+ if isinstance(r, BaseException):
336
+ final.append({"url": "?", "success": False, "error": str(r)})
337
+ else:
338
+ final.append(r)
339
+
340
+ if rejected:
341
+ for rej in rejected:
342
+ final.append({"url": rej["url"], "success": False, "content": "", "error": rej["error"]})
343
+
344
+ return json.dumps(final, ensure_ascii=False, indent=2, allow_nan=False)
345
+
346
+
347
+ if __name__ == "__main__":
348
+ from karon_mcp.__main__ import main
349
+ main()