crawlengine-mcp 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crawlengine_mcp-1.0.0/LICENSE +21 -0
- crawlengine_mcp-1.0.0/PKG-INFO +175 -0
- crawlengine_mcp-1.0.0/README.md +155 -0
- crawlengine_mcp-1.0.0/crawlengine/__init__.py +8 -0
- crawlengine_mcp-1.0.0/crawlengine/_genesis.py +11 -0
- crawlengine_mcp-1.0.0/crawlengine/cli.py +71 -0
- crawlengine_mcp-1.0.0/crawlengine/extractors/markdown.py +77 -0
- crawlengine_mcp-1.0.0/crawlengine/extractors/metadata.py +72 -0
- crawlengine_mcp-1.0.0/crawlengine/headers/rotator.py +53 -0
- crawlengine_mcp-1.0.0/crawlengine/server.py +222 -0
- crawlengine_mcp-1.0.0/crawlengine_mcp.egg-info/PKG-INFO +175 -0
- crawlengine_mcp-1.0.0/crawlengine_mcp.egg-info/SOURCES.txt +17 -0
- crawlengine_mcp-1.0.0/crawlengine_mcp.egg-info/dependency_links.txt +1 -0
- crawlengine_mcp-1.0.0/crawlengine_mcp.egg-info/entry_points.txt +3 -0
- crawlengine_mcp-1.0.0/crawlengine_mcp.egg-info/requires.txt +5 -0
- crawlengine_mcp-1.0.0/crawlengine_mcp.egg-info/top_level.txt +1 -0
- crawlengine_mcp-1.0.0/pyproject.toml +35 -0
- crawlengine_mcp-1.0.0/setup.cfg +4 -0
- crawlengine_mcp-1.0.0/tests/test_genesis_entrypoints.py +25 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Johann Schiffer / NEXUS
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: crawlengine-mcp
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Resilient Web Scraping & Clean Markdown Extraction MCP Server for AI Agents
|
|
5
|
+
Author: Johann Schiffer
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://johannschiffer.github.io
|
|
8
|
+
Project-URL: Repository, https://github.com/johannschiffer/crawlengine-mcp
|
|
9
|
+
Project-URL: Starter Kit, https://johanld.gumroad.com/l/genesis-starter-kit
|
|
10
|
+
Project-URL: Enterprise Suite, https://johanld.gumroad.com/l/genesis-enterprise-suite
|
|
11
|
+
Keywords: mcp,model-context-protocol,web-scraping,crawler,markdown,claude,cursor,ai-agent
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: httpx>=0.25.0
|
|
16
|
+
Requires-Dist: beautifulsoup4>=4.12.0
|
|
17
|
+
Provides-Extra: test
|
|
18
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# CrawlEngine MCP (Lite Edition)
|
|
22
|
+
|
|
23
|
+
<a href="https://glama.ai/mcp/servers/@johannschiffer/crawlengine-mcp">
|
|
24
|
+
<img width="380" height="200" src="https://glama.ai/mcp/servers/@johannschiffer/crawlengine-mcp/badge" alt="crawlengine-mcp MCP server" />
|
|
25
|
+
</a>
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
> **Free:** [Genesis Starter Kit](https://johanld.gumroad.com/l/genesis-starter-kit) (25 `.cursorrules`/`CLAUDE.md` rulesets + MCP configs). Try the [live config builder](https://johannschiffer.github.io).
|
|
29
|
+
> **Full stack:** all six servers in [Genesis Enterprise Suite ($19)](https://johanld.gumroad.com/l/genesis-enterprise-suite).
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
[](LICENSE)
|
|
33
|
+
[](https://python.org)
|
|
34
|
+
[](https://modelcontextprotocol.io)
|
|
35
|
+
|
|
36
|
+
**CrawlEngine MCP** is a resilient Model Context Protocol server that gives AI agents in **Claude Desktop**, **Cursor**, **Windsurf**, and **Antigravity** clean web-reading superpowers without choking context windows on raw HTML or getting instantly blocked by basic anti-bot firewalls.
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## Run without cloning
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
uvx --from git+https://github.com/johannschiffer/crawlengine-mcp crawlengine-mcp
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Claude Desktop / Cursor config entry:
|
|
47
|
+
|
|
48
|
+
```json
|
|
49
|
+
{
|
|
50
|
+
"mcpServers": {
|
|
51
|
+
"crawlengine-mcp": {
|
|
52
|
+
"command": "uvx",
|
|
53
|
+
"args": [
|
|
54
|
+
"--from",
|
|
55
|
+
"git+https://github.com/johannschiffer/crawlengine-mcp",
|
|
56
|
+
"crawlengine-mcp"
|
|
57
|
+
]
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
[](https://github.com/johannschiffer/crawlengine-mcp/actions/workflows/ci.yml)
|
|
64
|
+
|
|
65
|
+
## ๐ท๏ธ Why CrawlEngine MCP?
|
|
66
|
+
|
|
67
|
+
Most default web-fetching tools used by AI models suffer from severe drawbacks:
|
|
68
|
+
1. **Raw HTML Token Bloat:** A 150KB web page contains ~90% useless boilerplate (navbars, tracking pixels, CSS bundles, cookie banners) that burns LLM tokens and induces hallucinations.
|
|
69
|
+
2. **Instant HTTP 403 Blocks:** Default Python `urllib` or `requests` User-Agents trigger anti-bot WAFs immediately.
|
|
70
|
+
3. **Broken Relative Links:** Naive scrapers return unnavigable relative paths (`/docs/quickstart`) instead of canonical URLs.
|
|
71
|
+
|
|
72
|
+
**CrawlEngine** solves this by rotating realistic modern browser fingerprints and transforming messy DOM trees into pristine, semantic Markdown.
|
|
73
|
+
|
|
74
|
+
---
|
|
75
|
+
|
|
76
|
+
## โก Features (Lite Edition)
|
|
77
|
+
|
|
78
|
+
- ๐งน **Semantic Markdown Conversion**: Strips scripts, styles, popups, cookie consent overlays, and ads, keeping only the core article or documentation.
|
|
79
|
+
- ๐ญ **Realistic Browser Fingerprinting**: Automatically rotates `User-Agent`, `Sec-Ch-Ua`, and platform headers across Chrome, Safari, and Firefox.
|
|
80
|
+
- ๐ **Deep Link Discovery**: Extracts all outbound and internal links with anchor text.
|
|
81
|
+
- ๐ท๏ธ **Metadata Extraction**: Gathers OpenGraph, Twitter Cards, meta descriptions, and canonical URLs.
|
|
82
|
+
- ๐ชถ **Zero Complex Dependencies**: Pure Python using `httpx` and `beautifulsoup4`.
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## ๐ Quick Setup in 2 Minutes
|
|
87
|
+
|
|
88
|
+
### 1. Installation
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
git clone https://github.com/johannschiffer/crawlengine-mcp.git
|
|
92
|
+
cd crawlengine-mcp
|
|
93
|
+
pip install -r requirements.txt
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### 2. Configure Claude Desktop (`claude_desktop_config.json`)
|
|
97
|
+
|
|
98
|
+
On Linux: `~/.config/Claude/claude_desktop_config.json`
|
|
99
|
+
On macOS: `~/Library/Application Support/Claude/claude_desktop_config.json`
|
|
100
|
+
On Windows: `%APPDATA%\Claude\claude_desktop_config.json`
|
|
101
|
+
|
|
102
|
+
```json
|
|
103
|
+
{
|
|
104
|
+
"mcpServers": {
|
|
105
|
+
"crawlengine": {
|
|
106
|
+
"command": "python",
|
|
107
|
+
"args": ["-m", "crawlengine.server"],
|
|
108
|
+
"cwd": "/path/to/crawlengine-mcp"
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### 3. Verify via CLI
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
python -m crawlengine.cli scrape https://news.ycombinator.com
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
---
|
|
121
|
+
|
|
122
|
+
## ๐ ๏ธ MCP Tools Exposed to AI Agents
|
|
123
|
+
|
|
124
|
+
| Tool | Parameters | Description |
|
|
125
|
+
|---|---|---|
|
|
126
|
+
| `scrape_url` | `url`, `extract_markdown`, `timeout` | Fetches web page and extracts clean, token-efficient Markdown |
|
|
127
|
+
| `extract_links` | `url`, `internal_only` | Returns structured array of hyperlinks with anchor texts |
|
|
128
|
+
| `extract_metadata` | `url` | Retrieves page title, meta description, and OpenGraph data |
|
|
129
|
+
|
|
130
|
+
---
|
|
131
|
+
|
|
132
|
+
## ๐ Need Enterprise Anti-Bot Scraping? Upgrade to CrawlEngine Pro
|
|
133
|
+
|
|
134
|
+
Dealing with complex JavaScript SPAs, Cloudflare Turnstile, or scraping behind proxy pools?
|
|
135
|
+
|
|
136
|
+
| Feature | Lite (Open Source) | Enterprise Suite ($19) |
|
|
137
|
+
|---|:---:|:---:|
|
|
138
|
+
| **Semantic Markdown Extraction** | โ
| โ
|
|
|
139
|
+
| **Browser Fingerprint Rotation** | โ
| โ
|
|
|
140
|
+
| **OpenGraph & Link Extraction** | โ
| โ
|
|
|
141
|
+
| **Automatic Headless Browser Fallback** *(Playwright Chromium)* | โ | โ
|
|
|
142
|
+
| **JavaScript SPA Hydration** *(React, Next.js, Vue)* | โ | โ
|
|
|
143
|
+
| **Proxy Pool Rotation** *(HTTP & SOCKS5 with auto-ejection)* | โ | โ
|
|
|
144
|
+
| **Structured Data Extractor** *(JSON-LD, Pricing Tables, Products)* | โ | โ
|
|
|
145
|
+
| **Production Docker Container** | โ | โ
|
|
|
146
|
+
| **Full Pytest Suite & Anti-Detection Guide** | โ | โ
|
|
|
147
|
+
|
|
148
|
+
๐ **[Get the Genesis Enterprise Suite ($19)](https://johanld.gumroad.com/l/genesis-enterprise-suite)**
|
|
149
|
+
|
|
150
|
+
---
|
|
151
|
+
|
|
152
|
+
## ๐ License
|
|
153
|
+
|
|
154
|
+
MIT License. See [LICENSE](LICENSE) for details.
|
|
155
|
+
|
|
156
|
+
## Recommended Infrastructure
|
|
157
|
+
|
|
158
|
+
Proxies that work well with CrawlEngine MCP. Some links are referral links.
|
|
159
|
+
|
|
160
|
+
- [Webshare](https://www.webshare.io/)
|
|
161
|
+
- [ScraperAPI](https://www.scraperapi.com/)
|
|
162
|
+
- [Bright Data](https://brightdata.com/)
|
|
163
|
+
|
|
164
|
+
## Genesis MCP Ecosystem
|
|
165
|
+
|
|
166
|
+
| Module | What it does | Repo |
|
|
167
|
+
|---|---|---|
|
|
168
|
+
| **CrawlEngine MCP** (this repo) | Anti-bot web scraping, clean Markdown, proxy rotation and TLS evasion | [GitHub](https://github.com/johannschiffer/crawlengine-mcp) |
|
|
169
|
+
| **BrowserBridge MCP** | Deterministic headless browser automation with cryptographic action receipts | [GitHub](https://github.com/johannschiffer/browserbridge-mcp) |
|
|
170
|
+
| **PgSafe MCP** | Zero-trust PostgreSQL/SQLite guard, live PII masking, zero-downtime migration checks | [GitHub](https://github.com/johannschiffer/pgsafe-mcp) |
|
|
171
|
+
| **OmniBridge MCP** | Multi-channel notification and webhook bus with a persistent SQLite WAL queue | [GitHub](https://github.com/johannschiffer/omnibridge-mcp) |
|
|
172
|
+
| **SysFlow MCP** | Safe Linux administration plane: systemd, port audit, isolated execution | [GitHub](https://github.com/johannschiffer/sysflow-mcp) |
|
|
173
|
+
| **FastMCP Forge** | Production MCP server scaffolder: stdio, SSE and auth middleware | [GitHub](https://github.com/johannschiffer/fastmcp-forge) |
|
|
174
|
+
|
|
175
|
+
[Live config builder](https://johannschiffer.github.io) | [Starter Kit, free](https://johanld.gumroad.com/l/genesis-starter-kit) | [Enterprise Suite, $19](https://johanld.gumroad.com/l/genesis-enterprise-suite)
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# CrawlEngine MCP (Lite Edition)
|
|
2
|
+
|
|
3
|
+
<a href="https://glama.ai/mcp/servers/@johannschiffer/crawlengine-mcp">
|
|
4
|
+
<img width="380" height="200" src="https://glama.ai/mcp/servers/@johannschiffer/crawlengine-mcp/badge" alt="crawlengine-mcp MCP server" />
|
|
5
|
+
</a>
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
> **Free:** [Genesis Starter Kit](https://johanld.gumroad.com/l/genesis-starter-kit) (25 `.cursorrules`/`CLAUDE.md` rulesets + MCP configs). Try the [live config builder](https://johannschiffer.github.io).
|
|
9
|
+
> **Full stack:** all six servers in [Genesis Enterprise Suite ($19)](https://johanld.gumroad.com/l/genesis-enterprise-suite).
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
[](LICENSE)
|
|
13
|
+
[](https://python.org)
|
|
14
|
+
[](https://modelcontextprotocol.io)
|
|
15
|
+
|
|
16
|
+
**CrawlEngine MCP** is a resilient Model Context Protocol server that gives AI agents in **Claude Desktop**, **Cursor**, **Windsurf**, and **Antigravity** clean web-reading superpowers without choking context windows on raw HTML or getting instantly blocked by basic anti-bot firewalls.
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
## Run without cloning
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
uvx --from git+https://github.com/johannschiffer/crawlengine-mcp crawlengine-mcp
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Claude Desktop / Cursor config entry:
|
|
27
|
+
|
|
28
|
+
```json
|
|
29
|
+
{
|
|
30
|
+
"mcpServers": {
|
|
31
|
+
"crawlengine-mcp": {
|
|
32
|
+
"command": "uvx",
|
|
33
|
+
"args": [
|
|
34
|
+
"--from",
|
|
35
|
+
"git+https://github.com/johannschiffer/crawlengine-mcp",
|
|
36
|
+
"crawlengine-mcp"
|
|
37
|
+
]
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
[](https://github.com/johannschiffer/crawlengine-mcp/actions/workflows/ci.yml)
|
|
44
|
+
|
|
45
|
+
## ๐ท๏ธ Why CrawlEngine MCP?
|
|
46
|
+
|
|
47
|
+
Most default web-fetching tools used by AI models suffer from severe drawbacks:
|
|
48
|
+
1. **Raw HTML Token Bloat:** A 150KB web page contains ~90% useless boilerplate (navbars, tracking pixels, CSS bundles, cookie banners) that burns LLM tokens and induces hallucinations.
|
|
49
|
+
2. **Instant HTTP 403 Blocks:** Default Python `urllib` or `requests` User-Agents trigger anti-bot WAFs immediately.
|
|
50
|
+
3. **Broken Relative Links:** Naive scrapers return unnavigable relative paths (`/docs/quickstart`) instead of canonical URLs.
|
|
51
|
+
|
|
52
|
+
**CrawlEngine** solves this by rotating realistic modern browser fingerprints and transforming messy DOM trees into pristine, semantic Markdown.
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## โก Features (Lite Edition)
|
|
57
|
+
|
|
58
|
+
- ๐งน **Semantic Markdown Conversion**: Strips scripts, styles, popups, cookie consent overlays, and ads, keeping only the core article or documentation.
|
|
59
|
+
- ๐ญ **Realistic Browser Fingerprinting**: Automatically rotates `User-Agent`, `Sec-Ch-Ua`, and platform headers across Chrome, Safari, and Firefox.
|
|
60
|
+
- ๐ **Deep Link Discovery**: Extracts all outbound and internal links with anchor text.
|
|
61
|
+
- ๐ท๏ธ **Metadata Extraction**: Gathers OpenGraph, Twitter Cards, meta descriptions, and canonical URLs.
|
|
62
|
+
- ๐ชถ **Zero Complex Dependencies**: Pure Python using `httpx` and `beautifulsoup4`.
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
## ๐ Quick Setup in 2 Minutes
|
|
67
|
+
|
|
68
|
+
### 1. Installation
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
git clone https://github.com/johannschiffer/crawlengine-mcp.git
|
|
72
|
+
cd crawlengine-mcp
|
|
73
|
+
pip install -r requirements.txt
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
### 2. Configure Claude Desktop (`claude_desktop_config.json`)
|
|
77
|
+
|
|
78
|
+
On Linux: `~/.config/Claude/claude_desktop_config.json`
|
|
79
|
+
On macOS: `~/Library/Application Support/Claude/claude_desktop_config.json`
|
|
80
|
+
On Windows: `%APPDATA%\Claude\claude_desktop_config.json`
|
|
81
|
+
|
|
82
|
+
```json
|
|
83
|
+
{
|
|
84
|
+
"mcpServers": {
|
|
85
|
+
"crawlengine": {
|
|
86
|
+
"command": "python",
|
|
87
|
+
"args": ["-m", "crawlengine.server"],
|
|
88
|
+
"cwd": "/path/to/crawlengine-mcp"
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### 3. Verify via CLI
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
python -m crawlengine.cli scrape https://news.ycombinator.com
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## ๐ ๏ธ MCP Tools Exposed to AI Agents
|
|
103
|
+
|
|
104
|
+
| Tool | Parameters | Description |
|
|
105
|
+
|---|---|---|
|
|
106
|
+
| `scrape_url` | `url`, `extract_markdown`, `timeout` | Fetches web page and extracts clean, token-efficient Markdown |
|
|
107
|
+
| `extract_links` | `url`, `internal_only` | Returns structured array of hyperlinks with anchor texts |
|
|
108
|
+
| `extract_metadata` | `url` | Retrieves page title, meta description, and OpenGraph data |
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
## ๐ Need Enterprise Anti-Bot Scraping? Upgrade to CrawlEngine Pro
|
|
113
|
+
|
|
114
|
+
Dealing with complex JavaScript SPAs, Cloudflare Turnstile, or scraping behind proxy pools?
|
|
115
|
+
|
|
116
|
+
| Feature | Lite (Open Source) | Enterprise Suite ($19) |
|
|
117
|
+
|---|:---:|:---:|
|
|
118
|
+
| **Semantic Markdown Extraction** | โ
| โ
|
|
|
119
|
+
| **Browser Fingerprint Rotation** | โ
| โ
|
|
|
120
|
+
| **OpenGraph & Link Extraction** | โ
| โ
|
|
|
121
|
+
| **Automatic Headless Browser Fallback** *(Playwright Chromium)* | โ | โ
|
|
|
122
|
+
| **JavaScript SPA Hydration** *(React, Next.js, Vue)* | โ | โ
|
|
|
123
|
+
| **Proxy Pool Rotation** *(HTTP & SOCKS5 with auto-ejection)* | โ | โ
|
|
|
124
|
+
| **Structured Data Extractor** *(JSON-LD, Pricing Tables, Products)* | โ | โ
|
|
|
125
|
+
| **Production Docker Container** | โ | โ
|
|
|
126
|
+
| **Full Pytest Suite & Anti-Detection Guide** | โ | โ
|
|
|
127
|
+
|
|
128
|
+
๐ **[Get the Genesis Enterprise Suite ($19)](https://johanld.gumroad.com/l/genesis-enterprise-suite)**
|
|
129
|
+
|
|
130
|
+
---
|
|
131
|
+
|
|
132
|
+
## ๐ License
|
|
133
|
+
|
|
134
|
+
MIT License. See [LICENSE](LICENSE) for details.
|
|
135
|
+
|
|
136
|
+
## Recommended Infrastructure
|
|
137
|
+
|
|
138
|
+
Proxies that work well with CrawlEngine MCP. Some links are referral links.
|
|
139
|
+
|
|
140
|
+
- [Webshare](https://www.webshare.io/)
|
|
141
|
+
- [ScraperAPI](https://www.scraperapi.com/)
|
|
142
|
+
- [Bright Data](https://brightdata.com/)
|
|
143
|
+
|
|
144
|
+
## Genesis MCP Ecosystem
|
|
145
|
+
|
|
146
|
+
| Module | What it does | Repo |
|
|
147
|
+
|---|---|---|
|
|
148
|
+
| **CrawlEngine MCP** (this repo) | Anti-bot web scraping, clean Markdown, proxy rotation and TLS evasion | [GitHub](https://github.com/johannschiffer/crawlengine-mcp) |
|
|
149
|
+
| **BrowserBridge MCP** | Deterministic headless browser automation with cryptographic action receipts | [GitHub](https://github.com/johannschiffer/browserbridge-mcp) |
|
|
150
|
+
| **PgSafe MCP** | Zero-trust PostgreSQL/SQLite guard, live PII masking, zero-downtime migration checks | [GitHub](https://github.com/johannschiffer/pgsafe-mcp) |
|
|
151
|
+
| **OmniBridge MCP** | Multi-channel notification and webhook bus with a persistent SQLite WAL queue | [GitHub](https://github.com/johannschiffer/omnibridge-mcp) |
|
|
152
|
+
| **SysFlow MCP** | Safe Linux administration plane: systemd, port audit, isolated execution | [GitHub](https://github.com/johannschiffer/sysflow-mcp) |
|
|
153
|
+
| **FastMCP Forge** | Production MCP server scaffolder: stdio, SSE and auth middleware | [GitHub](https://github.com/johannschiffer/fastmcp-forge) |
|
|
154
|
+
|
|
155
|
+
[Live config builder](https://johannschiffer.github.io) | [Starter Kit, free](https://johanld.gumroad.com/l/genesis-starter-kit) | [Enterprise Suite, $19](https://johanld.gumroad.com/l/genesis-enterprise-suite)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Upgrade notice. Printed to stderr so MCP stdio traffic on stdout is never affected."""
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
NOTICE = 'CrawlEngine MCP (open edition). Starter Kit, free: https://johanld.gumroad.com/l/khffw | Enterprise Suite ($19): https://johanld.gumroad.com/l/tzuvz'
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def print_upgrade_notice():
|
|
9
|
+
if os.environ.get("GENESIS_NO_NOTICE") == "1":
|
|
10
|
+
return
|
|
11
|
+
print(NOTICE, file=sys.stderr)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CrawlEngine CLI testing utility.
|
|
3
|
+
Usage:
|
|
4
|
+
python -m crawlengine.cli scrape https://news.ycombinator.com
|
|
5
|
+
python -m crawlengine.cli links https://github.com
|
|
6
|
+
python -m crawlengine.cli meta https://dev.to
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import json
|
|
11
|
+
import sys
|
|
12
|
+
import httpx
|
|
13
|
+
from .headers.rotator import HeaderRotator
|
|
14
|
+
from .extractors.markdown import MarkdownExtractor
|
|
15
|
+
from .extractors.metadata import MetadataExtractor
|
|
16
|
+
|
|
17
|
+
def main():
|
|
18
|
+
parser = argparse.ArgumentParser(description="CrawlEngine CLI utility")
|
|
19
|
+
subparsers = parser.add_subparsers(dest="command", help="Sub-commands")
|
|
20
|
+
|
|
21
|
+
scrape_parser = subparsers.add_parser("scrape", help="Scrape and convert URL to Markdown")
|
|
22
|
+
scrape_parser.add_argument("url", help="URL to scrape")
|
|
23
|
+
|
|
24
|
+
links_parser = subparsers.add_parser("links", help="Extract links from URL")
|
|
25
|
+
links_parser.add_argument("url", help="URL to extract links from")
|
|
26
|
+
links_parser.add_argument("--internal-only", action="store_true", help="Filter to internal links")
|
|
27
|
+
|
|
28
|
+
meta_parser = subparsers.add_parser("meta", help="Extract OpenGraph and meta tags")
|
|
29
|
+
meta_parser.add_argument("url", help="URL to inspect")
|
|
30
|
+
|
|
31
|
+
args = parser.parse_args()
|
|
32
|
+
|
|
33
|
+
if not args.command:
|
|
34
|
+
parser.print_help()
|
|
35
|
+
sys.exit(0)
|
|
36
|
+
|
|
37
|
+
headers = HeaderRotator.get_headers(referer=args.url)
|
|
38
|
+
try:
|
|
39
|
+
with httpx.Client(follow_redirects=True, timeout=15.0) as client:
|
|
40
|
+
resp = client.get(args.url, headers=headers)
|
|
41
|
+
html = resp.text
|
|
42
|
+
except Exception as e:
|
|
43
|
+
print(f"Error fetching URL: {e}", file=sys.stderr)
|
|
44
|
+
sys.exit(1)
|
|
45
|
+
|
|
46
|
+
if args.command == "scrape":
|
|
47
|
+
md = MarkdownExtractor.html_to_markdown(html, base_url=args.url)
|
|
48
|
+
print(md)
|
|
49
|
+
|
|
50
|
+
elif args.command == "links":
|
|
51
|
+
links = MetadataExtractor.extract_links(html, base_url=args.url, internal_only=args.internal_only)
|
|
52
|
+
print(json.dumps(links, indent=2))
|
|
53
|
+
|
|
54
|
+
elif args.command == "meta":
|
|
55
|
+
meta = MetadataExtractor.extract(html, base_url=args.url)
|
|
56
|
+
print(json.dumps(meta, indent=2))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
_main_impl = main
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def main():
|
|
64
|
+
from ._genesis import print_upgrade_notice
|
|
65
|
+
try:
|
|
66
|
+
return _main_impl()
|
|
67
|
+
finally:
|
|
68
|
+
print_upgrade_notice()
|
|
69
|
+
|
|
70
|
+
if __name__ == "__main__":
|
|
71
|
+
main()
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""
|
|
2
|
+
HTML to Clean Semantic Markdown Extractor.
|
|
3
|
+
Strips boilerplate, navigation elements, tracking scripts, and cookie banners
|
|
4
|
+
to produce token-efficient, clean Markdown for LLMs.
|
|
5
|
+
"""
|
|
6
|
+
import re
|
|
7
|
+
from bs4 import BeautifulSoup
|
|
8
|
+
|
|
9
|
+
NOISE_TAGS = ["script", "style", "noscript", "iframe", "svg", "canvas", "nav", "footer", "aside"]
|
|
10
|
+
NOISE_CLASSES_IDS = re.compile(
|
|
11
|
+
r"(cookie|banner|advert|popup|modal|tracking|newsletter|social-share|sidebar|header|footer)",
|
|
12
|
+
re.IGNORECASE
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
class MarkdownExtractor:
|
|
16
|
+
@staticmethod
|
|
17
|
+
def html_to_markdown(html_content: str, base_url: str = "") -> str:
|
|
18
|
+
if not html_content or not html_content.strip():
|
|
19
|
+
return ""
|
|
20
|
+
|
|
21
|
+
soup = BeautifulSoup(html_content, "html.parser")
|
|
22
|
+
|
|
23
|
+
# 1. Remove noise tags
|
|
24
|
+
for tag in soup(NOISE_TAGS):
|
|
25
|
+
tag.decompose()
|
|
26
|
+
|
|
27
|
+
# 2. Remove noise divs/elements
|
|
28
|
+
for el in soup.find_all(attrs={"class": NOISE_CLASSES_IDS}):
|
|
29
|
+
# Don't delete main content container even if class has matching substring
|
|
30
|
+
if el.name in ["main", "article", "body"]:
|
|
31
|
+
continue
|
|
32
|
+
el.decompose()
|
|
33
|
+
|
|
34
|
+
for el in soup.find_all(attrs={"id": NOISE_CLASSES_IDS}):
|
|
35
|
+
if el.name in ["main", "article", "body"]:
|
|
36
|
+
continue
|
|
37
|
+
el.decompose()
|
|
38
|
+
|
|
39
|
+
# 3. Locate primary container if available
|
|
40
|
+
main_content = soup.find("main") or soup.find("article") or soup.find("body") or soup
|
|
41
|
+
|
|
42
|
+
markdown_lines = []
|
|
43
|
+
|
|
44
|
+
for element in main_content.find_all(["h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre", "blockquote"]):
|
|
45
|
+
text = element.get_text(strip=True)
|
|
46
|
+
if not text:
|
|
47
|
+
continue
|
|
48
|
+
|
|
49
|
+
tag_name = element.name
|
|
50
|
+
if tag_name == "h1":
|
|
51
|
+
markdown_lines.append(f"\n# {text}\n")
|
|
52
|
+
elif tag_name == "h2":
|
|
53
|
+
markdown_lines.append(f"\n## {text}\n")
|
|
54
|
+
elif tag_name == "h3":
|
|
55
|
+
markdown_lines.append(f"\n### {text}\n")
|
|
56
|
+
elif tag_name == "h4":
|
|
57
|
+
markdown_lines.append(f"\n#### {text}\n")
|
|
58
|
+
elif tag_name == "h5" or tag_name == "h6":
|
|
59
|
+
markdown_lines.append(f"\n##### {text}\n")
|
|
60
|
+
elif tag_name == "p":
|
|
61
|
+
markdown_lines.append(f"\n{text}\n")
|
|
62
|
+
elif tag_name == "li":
|
|
63
|
+
markdown_lines.append(f"- {text}")
|
|
64
|
+
elif tag_name == "pre":
|
|
65
|
+
markdown_lines.append(f"\n```\n{text}\n```\n")
|
|
66
|
+
elif tag_name == "blockquote":
|
|
67
|
+
markdown_lines.append(f"\n> {text}\n")
|
|
68
|
+
|
|
69
|
+
raw_md = "\n".join(markdown_lines)
|
|
70
|
+
# Collapse multiple blank lines
|
|
71
|
+
clean_md = re.sub(r"\n{3,}", "\n\n", raw_md).strip()
|
|
72
|
+
|
|
73
|
+
# Fallback if no specific tags matched
|
|
74
|
+
if not clean_md:
|
|
75
|
+
clean_md = main_content.get_text(separator="\n", strip=True)
|
|
76
|
+
|
|
77
|
+
return clean_md
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Web Metadata & Hyperlink Extractor.
|
|
3
|
+
Extracts title, meta tags, OpenGraph data, and structured links.
|
|
4
|
+
"""
|
|
5
|
+
from urllib.parse import urljoin, urlparse
|
|
6
|
+
from bs4 import BeautifulSoup
|
|
7
|
+
from typing import Dict, Any, List
|
|
8
|
+
|
|
9
|
+
class MetadataExtractor:
|
|
10
|
+
@staticmethod
|
|
11
|
+
def extract(html_content: str, base_url: str = "") -> Dict[str, Any]:
|
|
12
|
+
soup = BeautifulSoup(html_content, "html.parser")
|
|
13
|
+
|
|
14
|
+
# Title
|
|
15
|
+
title = ""
|
|
16
|
+
if soup.title and soup.title.string:
|
|
17
|
+
title = soup.title.string.strip()
|
|
18
|
+
|
|
19
|
+
# Meta tags
|
|
20
|
+
meta_desc = ""
|
|
21
|
+
desc_tag = soup.find("meta", attrs={"name": "description"}) or soup.find("meta", attrs={"property": "og:description"})
|
|
22
|
+
if desc_tag and desc_tag.get("content"):
|
|
23
|
+
meta_desc = desc_tag["content"].strip()
|
|
24
|
+
|
|
25
|
+
og_title = ""
|
|
26
|
+
og_tag = soup.find("meta", attrs={"property": "og:title"})
|
|
27
|
+
if og_tag and og_tag.get("content"):
|
|
28
|
+
og_title = og_tag["content"].strip()
|
|
29
|
+
|
|
30
|
+
canonical = ""
|
|
31
|
+
can_tag = soup.find("link", attrs={"rel": "canonical"})
|
|
32
|
+
if can_tag and can_tag.get("href"):
|
|
33
|
+
canonical = urljoin(base_url, can_tag["href"])
|
|
34
|
+
|
|
35
|
+
return {
|
|
36
|
+
"title": title or og_title,
|
|
37
|
+
"description": meta_desc,
|
|
38
|
+
"canonical_url": canonical,
|
|
39
|
+
"has_opengraph": bool(og_title or desc_tag)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
@staticmethod
|
|
43
|
+
def extract_links(html_content: str, base_url: str = "", internal_only: bool = False) -> List[Dict[str, str]]:
|
|
44
|
+
soup = BeautifulSoup(html_content, "html.parser")
|
|
45
|
+
base_domain = urlparse(base_url).netloc if base_url else ""
|
|
46
|
+
|
|
47
|
+
links = []
|
|
48
|
+
seen = set()
|
|
49
|
+
|
|
50
|
+
for a in soup.find_all("a", href=True):
|
|
51
|
+
href = a["href"].strip()
|
|
52
|
+
if not href or href.startswith("#") or href.startswith("javascript:") or href.startswith("mailto:"):
|
|
53
|
+
continue
|
|
54
|
+
|
|
55
|
+
full_url = urljoin(base_url, href)
|
|
56
|
+
target_domain = urlparse(full_url).netloc
|
|
57
|
+
|
|
58
|
+
if internal_only and base_domain and target_domain != base_domain:
|
|
59
|
+
continue
|
|
60
|
+
|
|
61
|
+
if full_url in seen:
|
|
62
|
+
continue
|
|
63
|
+
seen.add(full_url)
|
|
64
|
+
|
|
65
|
+
text = a.get_text(strip=True)
|
|
66
|
+
links.append({
|
|
67
|
+
"url": full_url,
|
|
68
|
+
"text": text[:100] if text else "",
|
|
69
|
+
"is_internal": (target_domain == base_domain) if base_domain else True
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
return links
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Browser Header & User-Agent Rotator.
|
|
3
|
+
Simulates real modern browsers across macOS, Linux, and Windows to bypass basic bot detection.
|
|
4
|
+
"""
|
|
5
|
+
import random
|
|
6
|
+
from typing import Dict
|
|
7
|
+
|
|
8
|
+
USER_AGENTS = [
|
|
9
|
+
# Chrome on macOS
|
|
10
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
|
|
11
|
+
# Chrome on Windows
|
|
12
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
|
|
13
|
+
# Firefox on Windows
|
|
14
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0",
|
|
15
|
+
# Safari on macOS
|
|
16
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
|
|
17
|
+
# Chrome on Linux
|
|
18
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
class HeaderRotator:
|
|
22
|
+
@staticmethod
|
|
23
|
+
def get_headers(referer: str = "https://www.google.com/") -> Dict[str, str]:
|
|
24
|
+
ua = random.choice(USER_AGENTS)
|
|
25
|
+
is_chrome = "Chrome" in ua
|
|
26
|
+
is_windows = "Windows" in ua
|
|
27
|
+
is_mac = "Macintosh" in ua
|
|
28
|
+
|
|
29
|
+
platform = '"Windows"' if is_windows else ('"macOS"' if is_mac else '"Linux"')
|
|
30
|
+
|
|
31
|
+
headers = {
|
|
32
|
+
"User-Agent": ua,
|
|
33
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
|
34
|
+
"Accept-Language": "en-US,en;q=0.9,es;q=0.8",
|
|
35
|
+
"Accept-Encoding": "gzip, deflate, br",
|
|
36
|
+
"DNT": "1",
|
|
37
|
+
"Connection": "keep-alive",
|
|
38
|
+
"Upgrade-Insecure-Requests": "1",
|
|
39
|
+
"Referer": referer
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
if is_chrome:
|
|
43
|
+
headers.update({
|
|
44
|
+
"Sec-Ch-Ua": '"Chromium";v="128", "Not;A=Brand";v="24", "Google Chrome";v="128"',
|
|
45
|
+
"Sec-Ch-Ua-Mobile": "?0",
|
|
46
|
+
"Sec-Ch-Ua-Platform": platform,
|
|
47
|
+
"Sec-Fetch-Dest": "document",
|
|
48
|
+
"Sec-Fetch-Mode": "navigate",
|
|
49
|
+
"Sec-Fetch-Site": "cross-site",
|
|
50
|
+
"Sec-Fetch-User": "?1"
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
return headers
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CrawlEngine MCP: Model Context Protocol Server (Lite Edition)
|
|
3
|
+
Implements standard JSON-RPC 2.0 stdio transport for web scraping and clean markdown extraction.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import sys
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
from typing import Dict, Any, List, Optional
|
|
10
|
+
import httpx
|
|
11
|
+
from .headers.rotator import HeaderRotator
|
|
12
|
+
from .extractors.markdown import MarkdownExtractor
|
|
13
|
+
from .extractors.metadata import MetadataExtractor
|
|
14
|
+
|
|
15
|
+
logging.basicConfig(
|
|
16
|
+
level=logging.INFO,
|
|
17
|
+
format="%(asctime)s [%(levelname)s] [CrawlEngine] %(message)s",
|
|
18
|
+
stream=sys.stderr
|
|
19
|
+
)
|
|
20
|
+
logger = logging.getLogger("CrawlEngineServer")
|
|
21
|
+
|
|
22
|
+
class CrawlEngineMCPServer:
|
|
23
|
+
def __init__(self):
|
|
24
|
+
self.server_name = "crawlengine-mcp"
|
|
25
|
+
self.server_version = "1.0.0"
|
|
26
|
+
|
|
27
|
+
def get_tools_definition(self) -> List[Dict[str, Any]]:
|
|
28
|
+
return [
|
|
29
|
+
{
|
|
30
|
+
"name": "scrape_url",
|
|
31
|
+
"description": "Scrape a web page with browser fingerprinting and convert its content into clean, token-efficient Markdown.",
|
|
32
|
+
"inputSchema": {
|
|
33
|
+
"type": "object",
|
|
34
|
+
"properties": {
|
|
35
|
+
"url": {"type": "string", "description": "Target HTTP/HTTPS web page URL"},
|
|
36
|
+
"extract_markdown": {"type": "boolean", "description": "Convert HTML into semantic Markdown (default: true)"},
|
|
37
|
+
"timeout": {"type": "integer", "description": "Request timeout in seconds (default: 15)"}
|
|
38
|
+
},
|
|
39
|
+
"required": ["url"]
|
|
40
|
+
}
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"name": "extract_links",
|
|
44
|
+
"description": "Extract all navigable hyperlinks, anchors, and references from a web page.",
|
|
45
|
+
"inputSchema": {
|
|
46
|
+
"type": "object",
|
|
47
|
+
"properties": {
|
|
48
|
+
"url": {"type": "string", "description": "Target web page URL"},
|
|
49
|
+
"internal_only": {"type": "boolean", "description": "Filter links to only those belonging to the same domain (default: false)"}
|
|
50
|
+
},
|
|
51
|
+
"required": ["url"]
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"name": "extract_metadata",
|
|
56
|
+
"description": "Retrieve page title, OpenGraph tags, meta descriptions, and canonical links.",
|
|
57
|
+
"inputSchema": {
|
|
58
|
+
"type": "object",
|
|
59
|
+
"properties": {
|
|
60
|
+
"url": {"type": "string", "description": "Target web page URL"}
|
|
61
|
+
},
|
|
62
|
+
"required": ["url"]
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
]
|
|
66
|
+
|
|
67
|
+
def _fetch_html(self, url: str, timeout: int = 15) -> tuple[int, str]:
|
|
68
|
+
headers = HeaderRotator.get_headers(referer=url)
|
|
69
|
+
with httpx.Client(follow_redirects=True, timeout=float(timeout)) as client:
|
|
70
|
+
resp = client.get(url, headers=headers)
|
|
71
|
+
return resp.status_code, resp.text
|
|
72
|
+
|
|
73
|
+
def handle_tool_call(self, name: str, arguments: Dict[str, Any]) -> Dict[str, Any]:
|
|
74
|
+
url = arguments.get("url", "").strip()
|
|
75
|
+
if not url.startswith("http://") and not url.startswith("https://"):
|
|
76
|
+
return {"success": False, "error": f"Invalid URL scheme. Must begin with http:// or https://: '{url}'"}
|
|
77
|
+
|
|
78
|
+
timeout = arguments.get("timeout", 15)
|
|
79
|
+
|
|
80
|
+
try:
|
|
81
|
+
status_code, html = self._fetch_html(url, timeout=timeout)
|
|
82
|
+
except Exception as e:
|
|
83
|
+
return {"success": False, "error": f"Network fetch error: {str(e)}", "url": url}
|
|
84
|
+
|
|
85
|
+
if status_code != 200:
|
|
86
|
+
return {
|
|
87
|
+
"success": False,
|
|
88
|
+
"status_code": status_code,
|
|
89
|
+
"error": f"Web server returned HTTP {status_code}",
|
|
90
|
+
"url": url
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
if name == "scrape_url":
|
|
94
|
+
extract_md = arguments.get("extract_markdown", True)
|
|
95
|
+
meta = MetadataExtractor.extract(html, base_url=url)
|
|
96
|
+
content = MarkdownExtractor.html_to_markdown(html, base_url=url) if extract_md else html[:10000]
|
|
97
|
+
|
|
98
|
+
return {
|
|
99
|
+
"success": True,
|
|
100
|
+
"url": url,
|
|
101
|
+
"status_code": status_code,
|
|
102
|
+
"title": meta.get("title", ""),
|
|
103
|
+
"description": meta.get("description", ""),
|
|
104
|
+
"markdown": content,
|
|
105
|
+
"length_chars": len(content)
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
elif name == "extract_links":
|
|
109
|
+
internal_only = arguments.get("internal_only", False)
|
|
110
|
+
links = MetadataExtractor.extract_links(html, base_url=url, internal_only=internal_only)
|
|
111
|
+
return {
|
|
112
|
+
"success": True,
|
|
113
|
+
"url": url,
|
|
114
|
+
"count": len(links),
|
|
115
|
+
"links": links
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
elif name == "extract_metadata":
|
|
119
|
+
meta = MetadataExtractor.extract(html, base_url=url)
|
|
120
|
+
return {
|
|
121
|
+
"success": True,
|
|
122
|
+
"url": url,
|
|
123
|
+
"metadata": meta
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
else:
|
|
127
|
+
raise ValueError(f"Unknown tool: {name}")
|
|
128
|
+
|
|
129
|
+
def handle_request(self, request: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
130
|
+
req_id = request.get("id")
|
|
131
|
+
method = request.get("method")
|
|
132
|
+
params = request.get("params", {})
|
|
133
|
+
|
|
134
|
+
if req_id is None:
|
|
135
|
+
if method == "notifications/initialized":
|
|
136
|
+
logger.info("Client completed initialization handshake.")
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
if method == "initialize":
|
|
140
|
+
return {
|
|
141
|
+
"jsonrpc": "2.0",
|
|
142
|
+
"id": req_id,
|
|
143
|
+
"result": {
|
|
144
|
+
"protocolVersion": "2024-11-05",
|
|
145
|
+
"capabilities": {"tools": {}},
|
|
146
|
+
"serverInfo": {
|
|
147
|
+
"name": self.server_name,
|
|
148
|
+
"version": self.server_version
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
elif method == "ping":
|
|
154
|
+
return {"jsonrpc": "2.0", "id": req_id, "result": {}}
|
|
155
|
+
|
|
156
|
+
elif method == "tools/list":
|
|
157
|
+
return {
|
|
158
|
+
"jsonrpc": "2.0",
|
|
159
|
+
"id": req_id,
|
|
160
|
+
"result": {"tools": self.get_tools_definition()}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
elif method == "tools/call":
|
|
164
|
+
tool_name = params.get("name")
|
|
165
|
+
tool_args = params.get("arguments", {})
|
|
166
|
+
try:
|
|
167
|
+
output = self.handle_tool_call(tool_name, tool_args)
|
|
168
|
+
return {
|
|
169
|
+
"jsonrpc": "2.0",
|
|
170
|
+
"id": req_id,
|
|
171
|
+
"result": {
|
|
172
|
+
"content": [
|
|
173
|
+
{
|
|
174
|
+
"type": "text",
|
|
175
|
+
"text": json.dumps(output, indent=2, ensure_ascii=False)
|
|
176
|
+
}
|
|
177
|
+
],
|
|
178
|
+
"isError": not output.get("success", True) if "success" in output else False
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
except Exception as e:
|
|
182
|
+
logger.error(f"Error calling tool {tool_name}: {e}")
|
|
183
|
+
return {
|
|
184
|
+
"jsonrpc": "2.0",
|
|
185
|
+
"id": req_id,
|
|
186
|
+
"error": {"code": -32000, "message": str(e)}
|
|
187
|
+
}
|
|
188
|
+
else:
|
|
189
|
+
return {
|
|
190
|
+
"jsonrpc": "2.0",
|
|
191
|
+
"id": req_id,
|
|
192
|
+
"error": {"code": -32601, "message": f"Method not found: {method}"}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
def run_stdio(self):
|
|
196
|
+
logger.info(f"Starting {self.server_name} v{self.server_version} stdio listener...")
|
|
197
|
+
for line in sys.stdin:
|
|
198
|
+
line = line.strip()
|
|
199
|
+
if not line:
|
|
200
|
+
continue
|
|
201
|
+
try:
|
|
202
|
+
request = json.loads(line)
|
|
203
|
+
response = self.handle_request(request)
|
|
204
|
+
if response is not None:
|
|
205
|
+
sys.stdout.write(json.dumps(response) + "\n")
|
|
206
|
+
sys.stdout.flush()
|
|
207
|
+
except Exception as e:
|
|
208
|
+
logger.error(f"Error processing stdio line: {e}")
|
|
209
|
+
|
|
210
|
+
def main():
|
|
211
|
+
import argparse
|
|
212
|
+
parser = argparse.ArgumentParser(
|
|
213
|
+
prog="crawlengine-mcp",
|
|
214
|
+
description="CrawlEngine MCP: Resilient Web Scraping & Clean Markdown Extraction for AI Agents"
|
|
215
|
+
)
|
|
216
|
+
parser.add_argument("-v", "--version", action="version", version="%(prog)s 1.0.0")
|
|
217
|
+
args, unknown = parser.parse_known_args()
|
|
218
|
+
server = CrawlEngineMCPServer()
|
|
219
|
+
server.run_stdio()
|
|
220
|
+
|
|
221
|
+
if __name__ == "__main__":
|
|
222
|
+
main()
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: crawlengine-mcp
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Resilient Web Scraping & Clean Markdown Extraction MCP Server for AI Agents
|
|
5
|
+
Author: Johann Schiffer
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://johannschiffer.github.io
|
|
8
|
+
Project-URL: Repository, https://github.com/johannschiffer/crawlengine-mcp
|
|
9
|
+
Project-URL: Starter Kit, https://johanld.gumroad.com/l/genesis-starter-kit
|
|
10
|
+
Project-URL: Enterprise Suite, https://johanld.gumroad.com/l/genesis-enterprise-suite
|
|
11
|
+
Keywords: mcp,model-context-protocol,web-scraping,crawler,markdown,claude,cursor,ai-agent
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: httpx>=0.25.0
|
|
16
|
+
Requires-Dist: beautifulsoup4>=4.12.0
|
|
17
|
+
Provides-Extra: test
|
|
18
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# CrawlEngine MCP (Lite Edition)
|
|
22
|
+
|
|
23
|
+
<a href="https://glama.ai/mcp/servers/@johannschiffer/crawlengine-mcp">
|
|
24
|
+
<img width="380" height="200" src="https://glama.ai/mcp/servers/@johannschiffer/crawlengine-mcp/badge" alt="crawlengine-mcp MCP server" />
|
|
25
|
+
</a>
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
> **Free:** [Genesis Starter Kit](https://johanld.gumroad.com/l/genesis-starter-kit) (25 `.cursorrules`/`CLAUDE.md` rulesets + MCP configs). Try the [live config builder](https://johannschiffer.github.io).
|
|
29
|
+
> **Full stack:** all six servers in [Genesis Enterprise Suite ($19)](https://johanld.gumroad.com/l/genesis-enterprise-suite).
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
[](LICENSE)
|
|
33
|
+
[](https://python.org)
|
|
34
|
+
[](https://modelcontextprotocol.io)
|
|
35
|
+
|
|
36
|
+
**CrawlEngine MCP** is a resilient Model Context Protocol server that gives AI agents in **Claude Desktop**, **Cursor**, **Windsurf**, and **Antigravity** clean web-reading superpowers without choking context windows on raw HTML or getting instantly blocked by basic anti-bot firewalls.
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## Run without cloning
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
uvx --from git+https://github.com/johannschiffer/crawlengine-mcp crawlengine-mcp
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Claude Desktop / Cursor config entry:
|
|
47
|
+
|
|
48
|
+
```json
|
|
49
|
+
{
|
|
50
|
+
"mcpServers": {
|
|
51
|
+
"crawlengine-mcp": {
|
|
52
|
+
"command": "uvx",
|
|
53
|
+
"args": [
|
|
54
|
+
"--from",
|
|
55
|
+
"git+https://github.com/johannschiffer/crawlengine-mcp",
|
|
56
|
+
"crawlengine-mcp"
|
|
57
|
+
]
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
[](https://github.com/johannschiffer/crawlengine-mcp/actions/workflows/ci.yml)
|
|
64
|
+
|
|
65
|
+
## ๐ท๏ธ Why CrawlEngine MCP?
|
|
66
|
+
|
|
67
|
+
Most default web-fetching tools used by AI models suffer from severe drawbacks:
|
|
68
|
+
1. **Raw HTML Token Bloat:** A 150KB web page contains ~90% useless boilerplate (navbars, tracking pixels, CSS bundles, cookie banners) that burns LLM tokens and induces hallucinations.
|
|
69
|
+
2. **Instant HTTP 403 Blocks:** Default Python `urllib` or `requests` User-Agents trigger anti-bot WAFs immediately.
|
|
70
|
+
3. **Broken Relative Links:** Naive scrapers return unnavigable relative paths (`/docs/quickstart`) instead of canonical URLs.
|
|
71
|
+
|
|
72
|
+
**CrawlEngine** solves this by rotating realistic modern browser fingerprints and transforming messy DOM trees into pristine, semantic Markdown.
|
|
73
|
+
|
|
74
|
+
---
|
|
75
|
+
|
|
76
|
+
## โก Features (Lite Edition)
|
|
77
|
+
|
|
78
|
+
- ๐งน **Semantic Markdown Conversion**: Strips scripts, styles, popups, cookie consent overlays, and ads, keeping only the core article or documentation.
|
|
79
|
+
- ๐ญ **Realistic Browser Fingerprinting**: Automatically rotates `User-Agent`, `Sec-Ch-Ua`, and platform headers across Chrome, Safari, and Firefox.
|
|
80
|
+
- ๐ **Deep Link Discovery**: Extracts all outbound and internal links with anchor text.
|
|
81
|
+
- ๐ท๏ธ **Metadata Extraction**: Gathers OpenGraph, Twitter Cards, meta descriptions, and canonical URLs.
|
|
82
|
+
- ๐ชถ **Zero Complex Dependencies**: Pure Python using `httpx` and `beautifulsoup4`.
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## ๐ Quick Setup in 2 Minutes
|
|
87
|
+
|
|
88
|
+
### 1. Installation
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
git clone https://github.com/johannschiffer/crawlengine-mcp.git
|
|
92
|
+
cd crawlengine-mcp
|
|
93
|
+
pip install -r requirements.txt
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### 2. Configure Claude Desktop (`claude_desktop_config.json`)
|
|
97
|
+
|
|
98
|
+
On Linux: `~/.config/Claude/claude_desktop_config.json`
|
|
99
|
+
On macOS: `~/Library/Application Support/Claude/claude_desktop_config.json`
|
|
100
|
+
On Windows: `%APPDATA%\Claude\claude_desktop_config.json`
|
|
101
|
+
|
|
102
|
+
```json
|
|
103
|
+
{
|
|
104
|
+
"mcpServers": {
|
|
105
|
+
"crawlengine": {
|
|
106
|
+
"command": "python",
|
|
107
|
+
"args": ["-m", "crawlengine.server"],
|
|
108
|
+
"cwd": "/path/to/crawlengine-mcp"
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### 3. Verify via CLI
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
python -m crawlengine.cli scrape https://news.ycombinator.com
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
---
|
|
121
|
+
|
|
122
|
+
## ๐ ๏ธ MCP Tools Exposed to AI Agents
|
|
123
|
+
|
|
124
|
+
| Tool | Parameters | Description |
|
|
125
|
+
|---|---|---|
|
|
126
|
+
| `scrape_url` | `url`, `extract_markdown`, `timeout` | Fetches web page and extracts clean, token-efficient Markdown |
|
|
127
|
+
| `extract_links` | `url`, `internal_only` | Returns structured array of hyperlinks with anchor texts |
|
|
128
|
+
| `extract_metadata` | `url` | Retrieves page title, meta description, and OpenGraph data |
|
|
129
|
+
|
|
130
|
+
---
|
|
131
|
+
|
|
132
|
+
## ๐ Need Enterprise Anti-Bot Scraping? Upgrade to CrawlEngine Pro
|
|
133
|
+
|
|
134
|
+
Dealing with complex JavaScript SPAs, Cloudflare Turnstile, or scraping behind proxy pools?
|
|
135
|
+
|
|
136
|
+
| Feature | Lite (Open Source) | Enterprise Suite ($19) |
|
|
137
|
+
|---|:---:|:---:|
|
|
138
|
+
| **Semantic Markdown Extraction** | โ
| โ
|
|
|
139
|
+
| **Browser Fingerprint Rotation** | โ
| โ
|
|
|
140
|
+
| **OpenGraph & Link Extraction** | โ
| โ
|
|
|
141
|
+
| **Automatic Headless Browser Fallback** *(Playwright Chromium)* | โ | โ
|
|
|
142
|
+
| **JavaScript SPA Hydration** *(React, Next.js, Vue)* | โ | โ
|
|
|
143
|
+
| **Proxy Pool Rotation** *(HTTP & SOCKS5 with auto-ejection)* | โ | โ
|
|
|
144
|
+
| **Structured Data Extractor** *(JSON-LD, Pricing Tables, Products)* | โ | โ
|
|
|
145
|
+
| **Production Docker Container** | โ | โ
|
|
|
146
|
+
| **Full Pytest Suite & Anti-Detection Guide** | โ | โ
|
|
|
147
|
+
|
|
148
|
+
๐ **[Get the Genesis Enterprise Suite ($19)](https://johanld.gumroad.com/l/genesis-enterprise-suite)**
|
|
149
|
+
|
|
150
|
+
---
|
|
151
|
+
|
|
152
|
+
## ๐ License
|
|
153
|
+
|
|
154
|
+
MIT License. See [LICENSE](LICENSE) for details.
|
|
155
|
+
|
|
156
|
+
## Recommended Infrastructure
|
|
157
|
+
|
|
158
|
+
Proxies that work well with CrawlEngine MCP. Some links are referral links.
|
|
159
|
+
|
|
160
|
+
- [Webshare](https://www.webshare.io/)
|
|
161
|
+
- [ScraperAPI](https://www.scraperapi.com/)
|
|
162
|
+
- [Bright Data](https://brightdata.com/)
|
|
163
|
+
|
|
164
|
+
## Genesis MCP Ecosystem
|
|
165
|
+
|
|
166
|
+
| Module | What it does | Repo |
|
|
167
|
+
|---|---|---|
|
|
168
|
+
| **CrawlEngine MCP** (this repo) | Anti-bot web scraping, clean Markdown, proxy rotation and TLS evasion | [GitHub](https://github.com/johannschiffer/crawlengine-mcp) |
|
|
169
|
+
| **BrowserBridge MCP** | Deterministic headless browser automation with cryptographic action receipts | [GitHub](https://github.com/johannschiffer/browserbridge-mcp) |
|
|
170
|
+
| **PgSafe MCP** | Zero-trust PostgreSQL/SQLite guard, live PII masking, zero-downtime migration checks | [GitHub](https://github.com/johannschiffer/pgsafe-mcp) |
|
|
171
|
+
| **OmniBridge MCP** | Multi-channel notification and webhook bus with a persistent SQLite WAL queue | [GitHub](https://github.com/johannschiffer/omnibridge-mcp) |
|
|
172
|
+
| **SysFlow MCP** | Safe Linux administration plane: systemd, port audit, isolated execution | [GitHub](https://github.com/johannschiffer/sysflow-mcp) |
|
|
173
|
+
| **FastMCP Forge** | Production MCP server scaffolder: stdio, SSE and auth middleware | [GitHub](https://github.com/johannschiffer/fastmcp-forge) |
|
|
174
|
+
|
|
175
|
+
[Live config builder](https://johannschiffer.github.io) | [Starter Kit, free](https://johanld.gumroad.com/l/genesis-starter-kit) | [Enterprise Suite, $19](https://johanld.gumroad.com/l/genesis-enterprise-suite)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
crawlengine/__init__.py
|
|
5
|
+
crawlengine/_genesis.py
|
|
6
|
+
crawlengine/cli.py
|
|
7
|
+
crawlengine/server.py
|
|
8
|
+
crawlengine/extractors/markdown.py
|
|
9
|
+
crawlengine/extractors/metadata.py
|
|
10
|
+
crawlengine/headers/rotator.py
|
|
11
|
+
crawlengine_mcp.egg-info/PKG-INFO
|
|
12
|
+
crawlengine_mcp.egg-info/SOURCES.txt
|
|
13
|
+
crawlengine_mcp.egg-info/dependency_links.txt
|
|
14
|
+
crawlengine_mcp.egg-info/entry_points.txt
|
|
15
|
+
crawlengine_mcp.egg-info/requires.txt
|
|
16
|
+
crawlengine_mcp.egg-info/top_level.txt
|
|
17
|
+
tests/test_genesis_entrypoints.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
crawlengine
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "crawlengine-mcp"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Resilient Web Scraping & Clean Markdown Extraction MCP Server for AI Agents"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Johann Schiffer"}
|
|
14
|
+
]
|
|
15
|
+
keywords = ["mcp", "model-context-protocol", "web-scraping", "crawler", "markdown", "claude", "cursor", "ai-agent"]
|
|
16
|
+
dependencies = [
|
|
17
|
+
"httpx>=0.25.0",
|
|
18
|
+
"beautifulsoup4>=4.12.0"
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.optional-dependencies]
|
|
22
|
+
test = ["pytest>=7"]
|
|
23
|
+
|
|
24
|
+
[project.scripts]
|
|
25
|
+
crawlengine-mcp = "crawlengine.server:main"
|
|
26
|
+
crawlengine-cli = "crawlengine.cli:main"
|
|
27
|
+
|
|
28
|
+
[tool.setuptools.packages.find]
|
|
29
|
+
include = ["crawlengine*"]
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://johannschiffer.github.io"
|
|
33
|
+
Repository = "https://github.com/johannschiffer/crawlengine-mcp"
|
|
34
|
+
"Starter Kit" = "https://johanld.gumroad.com/l/genesis-starter-kit"
|
|
35
|
+
"Enterprise Suite" = "https://johanld.gumroad.com/l/genesis-enterprise-suite"
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
import io
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def test_script_target_is_callable():
|
|
8
|
+
mod = importlib.import_module('crawlengine.server')
|
|
9
|
+
assert callable(getattr(mod, 'main'))
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_upgrade_notice_goes_to_stderr(capsys, monkeypatch):
|
|
13
|
+
monkeypatch.delenv('GENESIS_NO_NOTICE', raising=False)
|
|
14
|
+
from crawlengine._genesis import print_upgrade_notice
|
|
15
|
+
print_upgrade_notice()
|
|
16
|
+
out = capsys.readouterr()
|
|
17
|
+
assert out.out == ''
|
|
18
|
+
assert 'gumroad.com' in out.err
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_upgrade_notice_can_be_silenced(capsys, monkeypatch):
|
|
22
|
+
monkeypatch.setenv('GENESIS_NO_NOTICE', '1')
|
|
23
|
+
from crawlengine._genesis import print_upgrade_notice
|
|
24
|
+
print_upgrade_notice()
|
|
25
|
+
assert capsys.readouterr().err == ''
|