mcp-web-engine 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mcp_web_engine-1.1.0/.env.example +10 -0
- mcp_web_engine-1.1.0/.gitignore +14 -0
- mcp_web_engine-1.1.0/CONTRIBUTING.md +11 -0
- mcp_web_engine-1.1.0/Dockerfile +29 -0
- mcp_web_engine-1.1.0/LICENSE +21 -0
- mcp_web_engine-1.1.0/PKG-INFO +163 -0
- mcp_web_engine-1.1.0/README.md +142 -0
- mcp_web_engine-1.1.0/SECURITY.md +14 -0
- mcp_web_engine-1.1.0/bin/cli.js +90 -0
- mcp_web_engine-1.1.0/config.py +24 -0
- mcp_web_engine-1.1.0/docker-compose.yml +27 -0
- mcp_web_engine-1.1.0/docs/ONBOARDING.md +70 -0
- mcp_web_engine-1.1.0/docs/OUTREACH_PHASE_1.md +3 -0
- mcp_web_engine-1.1.0/logging_obs.py +111 -0
- mcp_web_engine-1.1.0/main.py +230 -0
- mcp_web_engine-1.1.0/manage_beta_keys.py +125 -0
- mcp_web_engine-1.1.0/mcp_protocol.py +210 -0
- mcp_web_engine-1.1.0/mcp_tools.py +50 -0
- mcp_web_engine-1.1.0/package.json +35 -0
- mcp_web_engine-1.1.0/pyproject.toml +44 -0
- mcp_web_engine-1.1.0/requirements.txt +9 -0
- mcp_web_engine-1.1.0/security.py +169 -0
- mcp_web_engine-1.1.0/src/mcp_web_engine/__init__.py +3 -0
- mcp_web_engine-1.1.0/src/mcp_web_engine/config.py +28 -0
- mcp_web_engine-1.1.0/src/mcp_web_engine/logging_obs.py +111 -0
- mcp_web_engine-1.1.0/src/mcp_web_engine/main.py +250 -0
- mcp_web_engine-1.1.0/src/mcp_web_engine/manage_beta_keys.py +125 -0
- mcp_web_engine-1.1.0/src/mcp_web_engine/mcp_protocol.py +216 -0
- mcp_web_engine-1.1.0/src/mcp_web_engine/mcp_tools.py +50 -0
- mcp_web_engine-1.1.0/src/mcp_web_engine/security.py +169 -0
- mcp_web_engine-1.1.0/src/mcp_web_engine/web_engine.py +200 -0
- mcp_web_engine-1.1.0/tests/handshake_stdio_test.py +96 -0
- mcp_web_engine-1.1.0/tests/test_suite.py +109 -0
- mcp_web_engine-1.1.0/web_engine.py +181 -0
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# MCP Web Engine Environment Configuration Example
|
|
2
|
+
API_KEY=sk_mcp_beta_example_key_change_me
|
|
3
|
+
SEARXNG_URL=http://host.docker.internal:8082/search
|
|
4
|
+
PORT=5050
|
|
5
|
+
HOST=0.0.0.0
|
|
6
|
+
RATE_LIMIT_PER_MINUTE=120
|
|
7
|
+
DEFAULT_TIMEOUT_SEC=10.0
|
|
8
|
+
MAX_PAYLOAD_BYTES=2097152
|
|
9
|
+
LOG_LEVEL=INFO
|
|
10
|
+
ENV=production
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Contributing to MCP Web Engine
|
|
2
|
+
|
|
3
|
+
Thank you for helping improve MCP Web Engine!
|
|
4
|
+
|
|
5
|
+
## Development Guidelines
|
|
6
|
+
|
|
7
|
+
1. **Fork & Clone:** Clone your fork locally.
|
|
8
|
+
2. **Virtual Environment:** Set up Python 3.11+ virtual environment (`python -m venv .venv && source .venv/bin/activate`).
|
|
9
|
+
3. **Dependencies:** Install dependencies with `pip install -r requirements.txt`.
|
|
10
|
+
4. **Security Rule:** Always ensure all SSRF validations pass (`validate_ssrf_url`).
|
|
11
|
+
5. **Run Tests:** Ensure 100% of tests pass before submitting a PR (`pytest tests/test_suite.py`).
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
FROM python:3.11-slim
|
|
2
|
+
|
|
3
|
+
# Create non-root user
|
|
4
|
+
RUN groupadd -g 1000 appgroup && \
|
|
5
|
+
useradd -u 1000 -g appgroup -s /bin/bash -m appuser
|
|
6
|
+
|
|
7
|
+
WORKDIR /app
|
|
8
|
+
|
|
9
|
+
# Install security updates & system dependencies
|
|
10
|
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
11
|
+
build-essential \
|
|
12
|
+
libssl-dev \
|
|
13
|
+
curl \
|
|
14
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
15
|
+
|
|
16
|
+
COPY requirements.txt .
|
|
17
|
+
RUN pip install --no-cache-dir -r requirements.txt
|
|
18
|
+
|
|
19
|
+
COPY --chown=appuser:appgroup . .
|
|
20
|
+
|
|
21
|
+
# Switch to non-root user
|
|
22
|
+
USER appuser
|
|
23
|
+
|
|
24
|
+
EXPOSE 5000
|
|
25
|
+
|
|
26
|
+
ENV PORT=5000
|
|
27
|
+
ENV HOST=0.0.0.0
|
|
28
|
+
|
|
29
|
+
CMD ["python", "main.py"]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 MCP Web Engine Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: mcp-web-engine
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Privacy-First, Self-Hostable & SSRF-Hardened MCP Server & Web Engine for AI Agents
|
|
5
|
+
Project-URL: Homepage, https://github.com/Arbolencio/mcp-web-engine
|
|
6
|
+
Project-URL: Repository, https://github.com/Arbolencio/mcp-web-engine.git
|
|
7
|
+
Author: Arbolencio
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: ai-agent,mcp,mcp-server,searxng,ssrf-protection,turbollm,web-scraper,web-search
|
|
11
|
+
Requires-Python: >=3.9
|
|
12
|
+
Requires-Dist: beautifulsoup4>=4.12.0
|
|
13
|
+
Requires-Dist: curl-cffi>=0.6.0
|
|
14
|
+
Requires-Dist: fastapi>=0.109.0
|
|
15
|
+
Requires-Dist: html2text>=2024.1.14
|
|
16
|
+
Requires-Dist: pydantic-settings>=2.1.0
|
|
17
|
+
Requires-Dist: pydantic>=2.5.0
|
|
18
|
+
Requires-Dist: sse-starlette>=2.0.0
|
|
19
|
+
Requires-Dist: uvicorn>=0.27.0
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# 🌐 MCP Web Engine & Search Gateway
|
|
23
|
+
|
|
24
|
+
> **Privacy-First, Self-Hostable & SSRF-Hardened MCP Server & Web Engine for AI Agents**
|
|
25
|
+
> **MCP Specification 2026-07-28 Pure Stateless Core & Stdio / HTTP Dual Mode**
|
|
26
|
+
|
|
27
|
+
[](https://pypi.org/project/mcp-web-engine/)
|
|
28
|
+
[](https://www.npmjs.com/package/mcp-web-engine)
|
|
29
|
+
[](LICENSE)
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## ⚡ Instant Quickstart with `uvx` (Recommended for AI Agents)
|
|
34
|
+
|
|
35
|
+
AI agents (Claude Desktop, Cursor, Windsurf, TurboLLM, Hermes Agent) can run `mcp-web-engine` in an isolated ephemeral environment with **zero configuration**:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
# Run stdio MCP server in an isolated environment (10ms startup, 0% system contamination)
|
|
39
|
+
uvx mcp-web-engine
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Or run via Git repository:
|
|
43
|
+
```bash
|
|
44
|
+
uvx --from git+https://github.com/Arbolencio/mcp-web-engine mcp-web-engine
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
## 📌 What it does & Why it exists
|
|
50
|
+
|
|
51
|
+
`mcp-web-engine` is a high-performance Model Context Protocol (MCP) server implementing the **MCP Specification 2026-07-28 Pure Stateless Core**. It provides AI Agents with secure, private, and SSRF-hardened access to multi-engine web search, raw content fetching, and clean HTML-to-Markdown extraction.
|
|
52
|
+
|
|
53
|
+
### Key Capabilities & Architecture
|
|
54
|
+
- **Dual Execution Mode:**
|
|
55
|
+
- **Stdio Mode (Default):** Runs JSON-RPC over `stdin`/`stdout` for direct LLM client integration (`uvx mcp-web-engine`).
|
|
56
|
+
- **HTTP/SSE Server Mode:** Runs FastAPI/Uvicorn server (`mcp-web-engine --serve`).
|
|
57
|
+
- **Pure Stateless Core (Spec 2026-07-28):** Zero sessions, zero state, zero initialize handshake required. Every request is completely independent.
|
|
58
|
+
- **Protocol Methods:** Implements `server/discover`, `tools/list`, and `tools/call` with standard JSON-RPC 2.0.
|
|
59
|
+
- **Zero Third-Party Tracking:** Powered by an internal SearXNG meta-search engine aggregating 70+ sources with DuckDuckGo fallback.
|
|
60
|
+
- **Automatic URL Normalization:** Handled transparently by `normalize_searxng_url()` — any provided `SEARXNG_URL` variation (`http://127.0.0.1:8082`, `http://127.0.0.1:8082/`, `http://127.0.0.1:8082/search`) automatically appends `/search` if missing.
|
|
61
|
+
- **SSRF Hardened:** Pre-request DNS resolution checks prevent agents from accessing internal networks (`localhost`, `127.0.0.1`, `192.168.x.x`, `169.254.169.254`). Step-by-step HTTP 301/302 redirect re-validation.
|
|
62
|
+
- **Zero `--break-system-packages`:** Executed 100% in sandboxed virtual environments created on-the-fly by `uv` or `venv`.
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
## 🔌 Integration Guides
|
|
67
|
+
|
|
68
|
+
### 1. TurboLLM / Local MCP Catalog
|
|
69
|
+
```typescript
|
|
70
|
+
{
|
|
71
|
+
id: 'mcp-web-engine',
|
|
72
|
+
name: 'MCP Web Engine',
|
|
73
|
+
cat: 'Search',
|
|
74
|
+
desc: 'Privacy-first, self-hostable & SSRF-hardened web search engine and scraper for AI agents.',
|
|
75
|
+
cmd: 'uvx mcp-web-engine==1.1.0',
|
|
76
|
+
uvx: true,
|
|
77
|
+
argNote: 'Requires a running SearXNG instance. No query leaves the host unless ALLOW_DDG_FALLBACK is enabled.',
|
|
78
|
+
envs: [
|
|
79
|
+
{ key: 'SEARXNG_URL', desc: 'SearXNG meta-search endpoint (default: http://127.0.0.1:8082/search)', required: true },
|
|
80
|
+
],
|
|
81
|
+
}
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### 2. Claude Desktop (`claude_desktop_config.json`)
|
|
85
|
+
```json
|
|
86
|
+
{
|
|
87
|
+
"mcpServers": {
|
|
88
|
+
"mcp-web-engine": {
|
|
89
|
+
"command": "uvx",
|
|
90
|
+
"args": ["mcp-web-engine"],
|
|
91
|
+
"env": {
|
|
92
|
+
"SEARXNG_URL": "http://127.0.0.1:8082/search"
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
### 3. Cursor / Windsurf / Hermes Agent
|
|
100
|
+
```json
|
|
101
|
+
{
|
|
102
|
+
"mcpServers": {
|
|
103
|
+
"mcp-web-engine": {
|
|
104
|
+
"command": "uvx",
|
|
105
|
+
"args": ["mcp-web-engine"]
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
113
|
+
## 🛠️ Provided MCP Tools
|
|
114
|
+
|
|
115
|
+
| Tool Name | Parameters | Description |
|
|
116
|
+
|---|---|---|
|
|
117
|
+
| `web_search` | `query` (str), `limit` (int, default: 10) | Multi-engine meta-search via SearXNG returning structured JSON results |
|
|
118
|
+
| `fetch_url` | `url` (str), `max_bytes` (int, optional) | Fetches raw web content after strict SSRF security validation |
|
|
119
|
+
| `extract_markdown` | `url` (str), `max_bytes` (int, optional) | Scrapes a webpage and converts HTML into clean, LLM-optimized Markdown |
|
|
120
|
+
|
|
121
|
+
---
|
|
122
|
+
|
|
123
|
+
## ⚙️ Configuration & Environment Variables
|
|
124
|
+
|
|
125
|
+
| Variable | Required | Default | Description |
|
|
126
|
+
|---|---|---|---|
|
|
127
|
+
| `SEARXNG_URL` | **Yes** | `http://127.0.0.1:8082/search` | SearXNG meta-search endpoint (Auto-normalizes trailing slashes and missing `/search`) |
|
|
128
|
+
| `ALLOW_DDG_FALLBACK` | No | `false` | **Privacy switch.** When `false` (default), a SearXNG failure returns `SEARXNG_UNAVAILABLE` instead of quietly querying DuckDuckGo. Set to `true` only if you accept that queries leave your host on failure. |
|
|
129
|
+
| `PORT` | No | `5050` | HTTP/SSE server port (Used only when running in `--serve` mode) |
|
|
130
|
+
| `HOST` | No | `0.0.0.0` | HTTP/SSE server binding host |
|
|
131
|
+
| `API_KEY` | No | `""` | Optional authentication key for HTTP endpoints |
|
|
132
|
+
|
|
133
|
+
---
|
|
134
|
+
|
|
135
|
+
## 🛠️ Development & Building
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
# Clone Repository
|
|
139
|
+
git clone https://github.com/Arbolencio/mcp-web-engine.git
|
|
140
|
+
cd mcp-web-engine
|
|
141
|
+
|
|
142
|
+
# Build Wheel for PyPI
|
|
143
|
+
python3 -m build --wheel
|
|
144
|
+
|
|
145
|
+
# Run locally in stdio mode via uv
|
|
146
|
+
uv run python -m mcp_web_engine.main
|
|
147
|
+
|
|
148
|
+
# Run locally in HTTP server mode via uv
|
|
149
|
+
uv run python -m mcp_web_engine.main --serve
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
---
|
|
153
|
+
|
|
154
|
+
## 🛡️ Security & Environment Isolation
|
|
155
|
+
|
|
156
|
+
- **SSRF Hardening:** Blocks `localhost`, loopbacks, private subnets (`10.0.0.0/8`, `192.168.0.0/16`, `172.16.0.0/12`), percent-encoding bypasses (`%31%32%37...`), and re-validates `Location` headers on HTTP redirects step-by-step.
|
|
157
|
+
- **Zero Package Pollution:** Execution via `uvx` or `bin/cli.js` never modifies global site-packages or uses dangerous flags like `--break-system-packages`.
|
|
158
|
+
|
|
159
|
+
---
|
|
160
|
+
|
|
161
|
+
## 📄 License
|
|
162
|
+
|
|
163
|
+
MIT License © 2026 MCP Web Engine Contributors.
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# 🌐 MCP Web Engine & Search Gateway
|
|
2
|
+
|
|
3
|
+
> **Privacy-First, Self-Hostable & SSRF-Hardened MCP Server & Web Engine for AI Agents**
|
|
4
|
+
> **MCP Specification 2026-07-28 Pure Stateless Core & Stdio / HTTP Dual Mode**
|
|
5
|
+
|
|
6
|
+
[](https://pypi.org/project/mcp-web-engine/)
|
|
7
|
+
[](https://www.npmjs.com/package/mcp-web-engine)
|
|
8
|
+
[](LICENSE)
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## ⚡ Instant Quickstart with `uvx` (Recommended for AI Agents)
|
|
13
|
+
|
|
14
|
+
AI agents (Claude Desktop, Cursor, Windsurf, TurboLLM, Hermes Agent) can run `mcp-web-engine` in an isolated ephemeral environment with **zero configuration**:
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
# Run stdio MCP server in an isolated environment (10ms startup, 0% system contamination)
|
|
18
|
+
uvx mcp-web-engine
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Or run via Git repository:
|
|
22
|
+
```bash
|
|
23
|
+
uvx --from git+https://github.com/Arbolencio/mcp-web-engine mcp-web-engine
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
## 📌 What it does & Why it exists
|
|
29
|
+
|
|
30
|
+
`mcp-web-engine` is a high-performance Model Context Protocol (MCP) server implementing the **MCP Specification 2026-07-28 Pure Stateless Core**. It provides AI Agents with secure, private, and SSRF-hardened access to multi-engine web search, raw content fetching, and clean HTML-to-Markdown extraction.
|
|
31
|
+
|
|
32
|
+
### Key Capabilities & Architecture
|
|
33
|
+
- **Dual Execution Mode:**
|
|
34
|
+
- **Stdio Mode (Default):** Runs JSON-RPC over `stdin`/`stdout` for direct LLM client integration (`uvx mcp-web-engine`).
|
|
35
|
+
- **HTTP/SSE Server Mode:** Runs FastAPI/Uvicorn server (`mcp-web-engine --serve`).
|
|
36
|
+
- **Pure Stateless Core (Spec 2026-07-28):** Zero sessions, zero state, zero initialize handshake required. Every request is completely independent.
|
|
37
|
+
- **Protocol Methods:** Implements `server/discover`, `tools/list`, and `tools/call` with standard JSON-RPC 2.0.
|
|
38
|
+
- **Zero Third-Party Tracking:** Powered by an internal SearXNG meta-search engine aggregating 70+ sources with DuckDuckGo fallback.
|
|
39
|
+
- **Automatic URL Normalization:** Handled transparently by `normalize_searxng_url()` — any provided `SEARXNG_URL` variation (`http://127.0.0.1:8082`, `http://127.0.0.1:8082/`, `http://127.0.0.1:8082/search`) automatically appends `/search` if missing.
|
|
40
|
+
- **SSRF Hardened:** Pre-request DNS resolution checks prevent agents from accessing internal networks (`localhost`, `127.0.0.1`, `192.168.x.x`, `169.254.169.254`). Step-by-step HTTP 301/302 redirect re-validation.
|
|
41
|
+
- **Zero `--break-system-packages`:** Executed 100% in sandboxed virtual environments created on-the-fly by `uv` or `venv`.
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## 🔌 Integration Guides
|
|
46
|
+
|
|
47
|
+
### 1. TurboLLM / Local MCP Catalog
|
|
48
|
+
```typescript
|
|
49
|
+
{
|
|
50
|
+
id: 'mcp-web-engine',
|
|
51
|
+
name: 'MCP Web Engine',
|
|
52
|
+
cat: 'Search',
|
|
53
|
+
desc: 'Privacy-first, self-hostable & SSRF-hardened web search engine and scraper for AI agents.',
|
|
54
|
+
cmd: 'uvx mcp-web-engine==1.1.0',
|
|
55
|
+
uvx: true,
|
|
56
|
+
argNote: 'Requires a running SearXNG instance. No query leaves the host unless ALLOW_DDG_FALLBACK is enabled.',
|
|
57
|
+
envs: [
|
|
58
|
+
{ key: 'SEARXNG_URL', desc: 'SearXNG meta-search endpoint (default: http://127.0.0.1:8082/search)', required: true },
|
|
59
|
+
],
|
|
60
|
+
}
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### 2. Claude Desktop (`claude_desktop_config.json`)
|
|
64
|
+
```json
|
|
65
|
+
{
|
|
66
|
+
"mcpServers": {
|
|
67
|
+
"mcp-web-engine": {
|
|
68
|
+
"command": "uvx",
|
|
69
|
+
"args": ["mcp-web-engine"],
|
|
70
|
+
"env": {
|
|
71
|
+
"SEARXNG_URL": "http://127.0.0.1:8082/search"
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
### 3. Cursor / Windsurf / Hermes Agent
|
|
79
|
+
```json
|
|
80
|
+
{
|
|
81
|
+
"mcpServers": {
|
|
82
|
+
"mcp-web-engine": {
|
|
83
|
+
"command": "uvx",
|
|
84
|
+
"args": ["mcp-web-engine"]
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
---
|
|
91
|
+
|
|
92
|
+
## 🛠️ Provided MCP Tools
|
|
93
|
+
|
|
94
|
+
| Tool Name | Parameters | Description |
|
|
95
|
+
|---|---|---|
|
|
96
|
+
| `web_search` | `query` (str), `limit` (int, default: 10) | Multi-engine meta-search via SearXNG returning structured JSON results |
|
|
97
|
+
| `fetch_url` | `url` (str), `max_bytes` (int, optional) | Fetches raw web content after strict SSRF security validation |
|
|
98
|
+
| `extract_markdown` | `url` (str), `max_bytes` (int, optional) | Scrapes a webpage and converts HTML into clean, LLM-optimized Markdown |
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## ⚙️ Configuration & Environment Variables
|
|
103
|
+
|
|
104
|
+
| Variable | Required | Default | Description |
|
|
105
|
+
|---|---|---|---|
|
|
106
|
+
| `SEARXNG_URL` | **Yes** | `http://127.0.0.1:8082/search` | SearXNG meta-search endpoint (Auto-normalizes trailing slashes and missing `/search`) |
|
|
107
|
+
| `ALLOW_DDG_FALLBACK` | No | `false` | **Privacy switch.** When `false` (default), a SearXNG failure returns `SEARXNG_UNAVAILABLE` instead of quietly querying DuckDuckGo. Set to `true` only if you accept that queries leave your host on failure. |
|
|
108
|
+
| `PORT` | No | `5050` | HTTP/SSE server port (Used only when running in `--serve` mode) |
|
|
109
|
+
| `HOST` | No | `0.0.0.0` | HTTP/SSE server binding host |
|
|
110
|
+
| `API_KEY` | No | `""` | Optional authentication key for HTTP endpoints |
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## 🛠️ Development & Building
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
# Clone Repository
|
|
118
|
+
git clone https://github.com/Arbolencio/mcp-web-engine.git
|
|
119
|
+
cd mcp-web-engine
|
|
120
|
+
|
|
121
|
+
# Build Wheel for PyPI
|
|
122
|
+
python3 -m build --wheel
|
|
123
|
+
|
|
124
|
+
# Run locally in stdio mode via uv
|
|
125
|
+
uv run python -m mcp_web_engine.main
|
|
126
|
+
|
|
127
|
+
# Run locally in HTTP server mode via uv
|
|
128
|
+
uv run python -m mcp_web_engine.main --serve
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
---
|
|
132
|
+
|
|
133
|
+
## 🛡️ Security & Environment Isolation
|
|
134
|
+
|
|
135
|
+
- **SSRF Hardening:** Blocks `localhost`, loopbacks, private subnets (`10.0.0.0/8`, `192.168.0.0/16`, `172.16.0.0/12`), percent-encoding bypasses (`%31%32%37...`), and re-validates `Location` headers on HTTP redirects step-by-step.
|
|
136
|
+
- **Zero Package Pollution:** Execution via `uvx` or `bin/cli.js` never modifies global site-packages or uses dangerous flags like `--break-system-packages`.
|
|
137
|
+
|
|
138
|
+
---
|
|
139
|
+
|
|
140
|
+
## 📄 License
|
|
141
|
+
|
|
142
|
+
MIT License © 2026 MCP Web Engine Contributors.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Security Policy
|
|
2
|
+
|
|
3
|
+
## Reporting Vulnerabilities
|
|
4
|
+
|
|
5
|
+
If you discover a security vulnerability (such as a potential SSRF bypass, authentication flaw, or denial of service), please **DO NOT** open a public issue.
|
|
6
|
+
|
|
7
|
+
Instead, please report it privately to the maintainers with a proof of concept.
|
|
8
|
+
|
|
9
|
+
## Security Features
|
|
10
|
+
|
|
11
|
+
- **SSRF Protection:** Pre-request DNS resolution verification blocking private subnets (`127.0.0.0/8`, `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `169.254.169.254`, IPv6 loopbacks, and percent-encoded IP bypasses).
|
|
12
|
+
- **Step-by-Step Redirect Re-Validation:** All HTTP 301/302 redirects are validated against SSRF prior to following.
|
|
13
|
+
- **Authentication:** All MCP tool endpoints require Bearer API key authorization.
|
|
14
|
+
- **Docker Hardening:** Runs under non-root UID 1000 with `no-new-privileges:true`.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* MCP Web Engine CLI Entry Point (npm / npx fallback)
|
|
5
|
+
* Spawns MCP Web Engine in an ISOLATED Python environment without polluting global system packages.
|
|
6
|
+
*
|
|
7
|
+
* Recommended execution for AI agents:
|
|
8
|
+
* uvx mcp-web-engine
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const { spawn, execSync } = require('child_process');
|
|
12
|
+
const path = require('path');
|
|
13
|
+
const fs = require('fs');
|
|
14
|
+
const os = require('os');
|
|
15
|
+
|
|
16
|
+
const reqTxt = path.join(__dirname, '..', 'requirements.txt');
|
|
17
|
+
|
|
18
|
+
function forwardChild(child) {
|
|
19
|
+
child.on('error', (err) => {
|
|
20
|
+
console.error('[mcp-web-engine] Failed to start process:', err.message);
|
|
21
|
+
process.exit(1);
|
|
22
|
+
});
|
|
23
|
+
child.on('exit', (code, signal) => {
|
|
24
|
+
if (signal) {
|
|
25
|
+
process.exit(128 + (signal === 'SIGINT' ? 2 : 15));
|
|
26
|
+
}
|
|
27
|
+
process.exit(code ?? 0);
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// 1. Try running with 'uv' if available (fastest, cleanest, isolated)
|
|
32
|
+
function tryUv() {
|
|
33
|
+
try {
|
|
34
|
+
execSync('uv --version', { stdio: 'ignore' });
|
|
35
|
+
const repoRoot = path.join(__dirname, '..');
|
|
36
|
+
const child = spawn('uv', ['run', '--directory', repoRoot, 'python', '-m', 'mcp_web_engine.main', ...process.argv.slice(2)], {
|
|
37
|
+
stdio: 'inherit',
|
|
38
|
+
env: process.env
|
|
39
|
+
});
|
|
40
|
+
forwardChild(child);
|
|
41
|
+
return true;
|
|
42
|
+
} catch (e) {
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// 2. Fallback to dedicated isolated virtualenv in user cache dir (~/.cache/mcp-web-engine/venv)
|
|
48
|
+
function runInIsolatedVenv() {
|
|
49
|
+
const cacheDir = process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache');
|
|
50
|
+
const venvDir = path.join(cacheDir, 'mcp-web-engine', 'venv');
|
|
51
|
+
const venvPython = process.platform === 'win32'
|
|
52
|
+
? path.join(venvDir, 'Scripts', 'python.exe')
|
|
53
|
+
: path.join(venvDir, 'bin', 'python');
|
|
54
|
+
const venvPip = process.platform === 'win32'
|
|
55
|
+
? path.join(venvDir, 'Scripts', 'pip.exe')
|
|
56
|
+
: path.join(venvDir, 'bin', 'pip');
|
|
57
|
+
|
|
58
|
+
function findSystemPython() {
|
|
59
|
+
for (const cmd of ['python3', 'python']) {
|
|
60
|
+
try {
|
|
61
|
+
execSync(`${cmd} --version`, { stdio: 'ignore' });
|
|
62
|
+
return cmd;
|
|
63
|
+
} catch (e) {}
|
|
64
|
+
}
|
|
65
|
+
throw new Error('Python 3 is required but was not found on PATH.');
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Ensure isolated venv exists and has requirements installed
|
|
69
|
+
if (!fs.existsSync(venvPython)) {
|
|
70
|
+
console.log('[mcp-web-engine] Creating isolated Python venv at:', venvDir);
|
|
71
|
+
const sysPython = findSystemPython();
|
|
72
|
+
fs.mkdirSync(path.dirname(venvDir), { recursive: true });
|
|
73
|
+
execSync(`${sysPython} -m venv "${venvDir}"`, { stdio: 'inherit' });
|
|
74
|
+
console.log('[mcp-web-engine] Installing dependencies into isolated venv...');
|
|
75
|
+
execSync(`"${venvPip}" install -r "${reqTxt}"`, { stdio: 'inherit' });
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const repoRoot = path.join(__dirname, '..');
|
|
79
|
+
const env = { ...process.env, PYTHONPATH: path.join(repoRoot, 'src') };
|
|
80
|
+
|
|
81
|
+
const child = spawn(venvPython, ['-m', 'mcp_web_engine.main', ...process.argv.slice(2)], {
|
|
82
|
+
stdio: 'inherit',
|
|
83
|
+
env: env
|
|
84
|
+
});
|
|
85
|
+
forwardChild(child);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
if (!tryUv()) {
|
|
89
|
+
runInIsolatedVenv();
|
|
90
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Centralized Configuration Module (Pydantic Settings / Environment Variables)
|
|
3
|
+
"""
|
|
4
|
+
from pydantic import Field
|
|
5
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
6
|
+
|
|
7
|
+
class Settings(BaseSettings):
|
|
8
|
+
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
|
|
9
|
+
|
|
10
|
+
API_KEY: str = Field(default="sk_mcp_dev_key_12345", description="Bearer API key for authentication")
|
|
11
|
+
SEARXNG_URL: str = Field(default="http://127.0.0.1:8082/search", description="Internal SearXNG URL")
|
|
12
|
+
PORT: int = Field(default=5050, description="Service listening port")
|
|
13
|
+
HOST: str = Field(default="0.0.0.0", description="Service listening host")
|
|
14
|
+
|
|
15
|
+
# Rate Limiting & Limits
|
|
16
|
+
RATE_LIMIT_PER_MINUTE: int = Field(default=120, description="Max requests per minute per API key")
|
|
17
|
+
DEFAULT_TIMEOUT_SEC: float = Field(default=10.0, description="Default HTTP request timeout in seconds")
|
|
18
|
+
MAX_PAYLOAD_BYTES: int = Field(default=2 * 1024 * 1024, description="Max payload size limit (2 MB)")
|
|
19
|
+
|
|
20
|
+
# Environment & Logging
|
|
21
|
+
ENV: str = Field(default="production", description="Environment (development/production)")
|
|
22
|
+
LOG_LEVEL: str = Field(default="INFO", description="Log level")
|
|
23
|
+
|
|
24
|
+
settings = Settings()
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
version: '3.8'
|
|
2
|
+
|
|
3
|
+
services:
|
|
4
|
+
mcp-web-engine:
|
|
5
|
+
build:
|
|
6
|
+
context: .
|
|
7
|
+
dockerfile: Dockerfile
|
|
8
|
+
container_name: mcp_web_engine
|
|
9
|
+
ports:
|
|
10
|
+
- "5050:5050"
|
|
11
|
+
extra_hosts:
|
|
12
|
+
- "host.docker.internal:host-gateway"
|
|
13
|
+
environment:
|
|
14
|
+
- API_KEY=sk_mcp_dev_key_12345
|
|
15
|
+
- SEARXNG_URL=http://host.docker.internal:8082/search
|
|
16
|
+
- PORT=5050
|
|
17
|
+
- HOST=0.0.0.0
|
|
18
|
+
- RATE_LIMIT_PER_MINUTE=120
|
|
19
|
+
- LOG_LEVEL=INFO
|
|
20
|
+
restart: unless-stopped
|
|
21
|
+
security_opt:
|
|
22
|
+
- no-new-privileges:true
|
|
23
|
+
healthcheck:
|
|
24
|
+
test: ["CMD", "curl", "-f", "http://localhost:5050/health"]
|
|
25
|
+
interval: 15s
|
|
26
|
+
timeout: 5s
|
|
27
|
+
retries: 3
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# 🚀 Guía de Onboarding Beta Pública — MCP Web Engine
|
|
2
|
+
|
|
3
|
+
> **Guía rápida de 1 página para conectar tu cliente MCP (Claude Desktop, Cursor, Windsurf, Inspector) al endpoint HTTPS público en < 3 minutos.**
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 🌐 Endpoint Público de Producción HTTPS (Cloudflare Managed Tunnel)
|
|
8
|
+
|
|
9
|
+
- **Endpoint MCP 2026-07-28:** `https://ultimate-ignored-over-light.trycloudflare.com/v1/mcp`
|
|
10
|
+
- **Health Check:** `https://ultimate-ignored-over-light.trycloudflare.com/health`
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
## 🔑 Paso 1: Obtén tu Beta Key
|
|
15
|
+
|
|
16
|
+
Contacta con el administrador para recibir tu clave personal e independiente de acceso Beta (formato `sk_mcp_beta_...`).
|
|
17
|
+
Cada clave Beta cuenta con un límite de **10,000 peticiones** y una tasa máxima de **120 req/minuto**.
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## ⚙️ Paso 2: Configura tu Cliente MCP
|
|
22
|
+
|
|
23
|
+
### A. Configuración en Cursor / Windsurf / MCP Inspector (Conexión Directa HTTPS)
|
|
24
|
+
1. Abre los ajustes de **MCP / Custom Tools** en tu IDE.
|
|
25
|
+
2. Añade un nuevo servidor con la siguiente URL:
|
|
26
|
+
- **URL:** `https://ultimate-ignored-over-light.trycloudflare.com/v1/mcp`
|
|
27
|
+
3. Añade la cabecera HTTP de autenticación:
|
|
28
|
+
- **Header Name:** `Authorization`
|
|
29
|
+
- **Header Value:** `Bearer sk_mcp_beta_TU_CLAVE_AQUÍ`
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
### B. Configuración en Claude Desktop (vía puente stdio `mcp-remote`)
|
|
34
|
+
Abre tu archivo `claude_desktop_config.json` y añade:
|
|
35
|
+
|
|
36
|
+
```json
|
|
37
|
+
{
|
|
38
|
+
"mcpServers": {
|
|
39
|
+
"mcp-web-engine": {
|
|
40
|
+
"command": "npx",
|
|
41
|
+
"args": [
|
|
42
|
+
"-y",
|
|
43
|
+
"mcp-remote",
|
|
44
|
+
"https://ultimate-ignored-over-light.trycloudflare.com/v1/mcp",
|
|
45
|
+
"--header",
|
|
46
|
+
"Authorization: Bearer sk_mcp_beta_TU_CLAVE_AQUÍ"
|
|
47
|
+
]
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## 🧪 Paso 3: Prueba tu Primera Invocación MCP
|
|
56
|
+
|
|
57
|
+
Una vez conectado, pídele a tu modelo/asistente:
|
|
58
|
+
|
|
59
|
+
> *"Busca los últimos avances del protocolo MCP en 2026 y extrae un resumen en Markdown."*
|
|
60
|
+
|
|
61
|
+
Tu asistente ejecutará de forma transparente:
|
|
62
|
+
1. `web_search(query="MCP protocol 2026 advancements")`
|
|
63
|
+
2. `extract_markdown(url="https://...")`
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## 🔒 Privacidad y Seguridad
|
|
68
|
+
|
|
69
|
+
- **Zero Tracking:** No almacenamos tus consultas de búsqueda ni el contenido descargado.
|
|
70
|
+
- **Protección Anti-SSRF:** La infraestructura bloquea el acceso accidental o malicioso a redes privadas locales.
|