pyrecrawl 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyrecrawl-0.2.1/.github/workflows/publish.yml +33 -0
- pyrecrawl-0.2.1/.gitignore +18 -0
- pyrecrawl-0.2.1/LICENSE +21 -0
- pyrecrawl-0.2.1/PKG-INFO +217 -0
- pyrecrawl-0.2.1/README.md +182 -0
- pyrecrawl-0.2.1/pyproject.toml +59 -0
- pyrecrawl-0.2.1/scripts/probe_stdio.py +114 -0
- pyrecrawl-0.2.1/scripts/selfcheck.py +57 -0
- pyrecrawl-0.2.1/src/pyrecrawl/__init__.py +9 -0
- pyrecrawl-0.2.1/src/pyrecrawl/cli.py +382 -0
- pyrecrawl-0.2.1/src/pyrecrawl/engines.py +685 -0
- pyrecrawl-0.2.1/src/pyrecrawl/server.py +269 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
name: publish
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read # checkout@v4 needs this; partial permissions block = everything else none
|
|
10
|
+
id-token: write # PyPI trusted publishing (OIDC)
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
build:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- uses: astral-sh/setup-uv@v6
|
|
18
|
+
- run: uv build # sdist + wheel into dist/ via isolated hatchling env
|
|
19
|
+
- uses: actions/upload-artifact@v4
|
|
20
|
+
with:
|
|
21
|
+
name: dist
|
|
22
|
+
path: dist/
|
|
23
|
+
|
|
24
|
+
publish:
|
|
25
|
+
needs: build
|
|
26
|
+
runs-on: ubuntu-latest
|
|
27
|
+
environment: pypi
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/download-artifact@v4
|
|
30
|
+
with:
|
|
31
|
+
name: dist
|
|
32
|
+
path: dist/
|
|
33
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
pyrecrawl-0.2.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Alexander Jay
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pyrecrawl-0.2.1/PKG-INFO
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pyrecrawl
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: Self-hosted Firecrawl alternative: LLM-ready scraper/crawler MCP server combining Crawl4AI and Scrapling with a smart auto-fallback ladder.
|
|
5
|
+
Project-URL: Homepage, https://github.com/SanggonBoy/PyreCrawl
|
|
6
|
+
Project-URL: Repository, https://github.com/SanggonBoy/PyreCrawl
|
|
7
|
+
Project-URL: Documentation, https://github.com/SanggonBoy/PyreCrawl#readme
|
|
8
|
+
Project-URL: Issues, https://github.com/SanggonBoy/PyreCrawl/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/SanggonBoy/PyreCrawl/releases
|
|
10
|
+
Author: Alexander Jay
|
|
11
|
+
License: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: ai-agent,ai-agent-tools,crawl4ai,firecrawl-alternative,llm,mcp,rag,scraping,scrapling,web-scraping
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Requires-Dist: crawl4ai>=0.9.0
|
|
26
|
+
Requires-Dist: mcp<2.0,>=1.0
|
|
27
|
+
Requires-Dist: pydantic>=2.0
|
|
28
|
+
Requires-Dist: scrapling[fetchers]>=0.4.0
|
|
29
|
+
Provides-Extra: browsers
|
|
30
|
+
Requires-Dist: playwright>=1.40; extra == 'browsers'
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
33
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
# 🔥 PyreCrawl — Self-Hosted Firecrawl Alternative as an MCP Server
|
|
37
|
+
|
|
38
|
+
[](https://opensource.org/licenses/MIT)
|
|
39
|
+
[](https://modelcontextprotocol.io/)
|
|
40
|
+
[](https://www.python.org/)
|
|
41
|
+
[](https://pypi.org/project/pyrecrawl/)
|
|
42
|
+
|
|
43
|
+
A **self-hosted Firecrawl alternative** combining two best-in-class open-source scrapers
|
|
44
|
+
behind a single Model Context Protocol (MCP) server:
|
|
45
|
+
|
|
46
|
+
- **[Scrapling](https://github.com/D4Vinci/Scrapling)** — fast HTTP (curl_cffi) + stealth browser with **Cloudflare Turnstile bypass**, adaptive element tracking, and XHR capture
|
|
47
|
+
- **[Crawl4AI](https://github.com/unclecode/crawl4ai)** — LLM-first crawler: BM25 fit-markdown, citations, structured extraction, deep crawl (BFS/DFS/BestFirst)
|
|
48
|
+
|
|
49
|
+
A **smart auto-fallback ladder** tries the cheapest engine that succeeds:
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
fast HTTP (Scrapling)
|
|
53
|
+
│ (403/503/Cloudflare challenge or empty body)
|
|
54
|
+
▼
|
|
55
|
+
stealth browser (Scrapling StealthyFetcher + CF solver)
|
|
56
|
+
│ (still blocked or page needs full JS rendering)
|
|
57
|
+
▼
|
|
58
|
+
full LLM processing (Crawl4AI AsyncWebCrawler + BM25)
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## ⚡ Tools exposed
|
|
62
|
+
|
|
63
|
+
| Tool | What it does |
|
|
64
|
+
|---|---|
|
|
65
|
+
| `scrape(url, prefer="auto")` | Single URL → LLM-ready markdown |
|
|
66
|
+
| `extract(url, schema)` | Scrape + structured extraction (JsonCss schema) |
|
|
67
|
+
| `map_site(root, include_pattern=None, limit=200)` | Enumerate all internal URLs |
|
|
68
|
+
| `crawl(root, max_pages=5, prefer="auto")` | Multi-page crawl with auto-fallback per page |
|
|
69
|
+
| `search(query, limit=10)` | Web search via DuckDuckGo HTML (no API key) |
|
|
70
|
+
| `health()` | Versions + import sanity check |
|
|
71
|
+
|
|
72
|
+
`prefer` options: `"auto"` (default ladder) · `"fast"` (HTTP only) · `"stealth"` (CF bypass) · `"llm"` (full Crawl4AI).
|
|
73
|
+
|
|
74
|
+
---
|
|
75
|
+
|
|
76
|
+
## 🚀 Install & Use (one-liner)
|
|
77
|
+
|
|
78
|
+
### 1. Install
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
# Using uv (recommended — fast, isolated, no venv needed)
|
|
82
|
+
uv tool install pyrecrawl
|
|
83
|
+
|
|
84
|
+
# Or pipx (alternative)
|
|
85
|
+
pipx install pyrecrawl
|
|
86
|
+
|
|
87
|
+
# Or pip into a venv
|
|
88
|
+
pip install pyrecrawl
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### 2. One-time browser engines
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
pyrecrawl setup
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
This installs Playwright Chromium + Scrapling engines (~2 min, one-time).
|
|
98
|
+
|
|
99
|
+
### 3. Register with your AI agent
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
# Auto-detect installed agents and write their MCP configs
|
|
103
|
+
pyrecrawl install
|
|
104
|
+
|
|
105
|
+
# Or target specific agents
|
|
106
|
+
pyrecrawl install claude-desktop cursor
|
|
107
|
+
|
|
108
|
+
# Dry-run to preview what would change
|
|
109
|
+
pyrecrawl install --dry-run
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Supported agents: `claude-desktop`, `claude-code`, `cursor`, `vscode`, `codex`, `opencode`, `hermes`.
|
|
113
|
+
|
|
114
|
+
### 4. Start chatting
|
|
115
|
+
|
|
116
|
+
After installing + registering, **restart your agent** (or start a new session). Then ask:
|
|
117
|
+
|
|
118
|
+
> *"Scrape https://example.com and summarize it."*
|
|
119
|
+
|
|
120
|
+
The tools appear as `mcp_pyrecrawl_scrape`, `mcp_pyrecrawl_extract`, `mcp_pyrecrawl_map_site`, `mcp_pyrecrawl_crawl`, `mcp_pyrecrawl_search`, `mcp_pyrecrawl_health`.
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
## 📚 Manual config (if `pyrecrawl install` doesn't match your setup)
|
|
125
|
+
|
|
126
|
+
| Agent | Config file | Entry |
|
|
127
|
+
|---|---|---|
|
|
128
|
+
| **Claude Desktop** | `~/.config/Claude/claude_desktop_config.json` (Linux) / `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) / `%AppData%\Claude\claude_desktop_config.json` (Windows) | `{"mcpServers":{"pyrecrawl":{"command":"uvx","args":["--from","pyrecrawl","pyrecrawl","serve"]}}}` |
|
|
129
|
+
| **Claude Code** | project-scoped `.mcp.json` | same JSON as Claude Desktop |
|
|
130
|
+
| **Cursor** | `~/.cursor/mcp.json` | same JSON |
|
|
131
|
+
| **VS Code / Copilot** | `.vscode/mcp.json` (project) | `{"servers":{"pyrecrawl":{"command":"uvx","args":["--from","pyrecrawl","pyrecrawl","serve"],"type":"stdio"}}}` |
|
|
132
|
+
| **Codex CLI** | `~/.codex/config.toml` | `[mcp_servers.pyrecrawl]\ncommand = "uvx"\nargs = ["--from", "pyrecrawl", "pyrecrawl", "serve"]` |
|
|
133
|
+
| **OpenCode** | `~/.config/opencode/opencode.json` | `{"mcp":{"pyrecrawl":{"type":"local","command":["uvx","--from","pyrecrawl","pyrecrawl","serve"],"enabled":true}}}` |
|
|
134
|
+
| **Hermes** | `~/.hermes/config.yaml` or `%LocalAppData%\hermes\config.yaml` | `mcp_servers:\n pyrecrawl:\n command: uvx\n args:\n - --from\n - pyrecrawl\n - pyrecrawl\n - serve\n enabled: true` |
|
|
135
|
+
|
|
136
|
+
> **Windows note:** `uvx` must be on PATH. If not, use the full path to `uvx.exe` (e.g. `C:\Users\<you>\AppData\Local\hermes\bin\uvx.exe`).
|
|
137
|
+
|
|
138
|
+
---
|
|
139
|
+
|
|
140
|
+
## 🧠 Why two engines?
|
|
141
|
+
|
|
142
|
+
| Concern | Scrapling | Crawl4AI |
|
|
143
|
+
|---|---|---|
|
|
144
|
+
| Static HTML page | ✅ curl_cffi, ~200ms | ✅ browser overhead, ~3s |
|
|
145
|
+
| Cloudflare-protected | ✅ Turnstile solver | ⚠️ requires stealth setup |
|
|
146
|
+
| JS-heavy SPA | ✅ Chromium real browser | ✅ same |
|
|
147
|
+
| LLM-ready markdown | ⚠️ basic | ✅ BM25 + citations + fit |
|
|
148
|
+
| Structured extraction (CSS schema) | ❌ | ✅ JsonCss strategy |
|
|
149
|
+
| Deep crawl (BFS/DFS/BestFirst) | ✅ Spider + AutoThrottle | ✅ BFS/DFS/BestFirst + adaptive |
|
|
150
|
+
| Adaptive element tracking | ✅ parser relocates moved elements | ❌ |
|
|
151
|
+
|
|
152
|
+
**PyreCrawl = Scrapling for fetch & bypass + Crawl4AI for processing & extraction.**
|
|
153
|
+
|
|
154
|
+
---
|
|
155
|
+
|
|
156
|
+
## 📊 Compared to Firecrawl (hosted)
|
|
157
|
+
|
|
158
|
+
| | Firecrawl | PyreCrawl |
|
|
159
|
+
|---|---|---|
|
|
160
|
+
| Cost | Free 1k/mo, then $16–333/mo | **Free, self-hosted** |
|
|
161
|
+
| Local LLM support | ❌ | ✅ Ollama / any LLM |
|
|
162
|
+
| Cloudflare bypass | ✅ (Fire-Engine, paid) | ✅ (free, Scrapling) |
|
|
163
|
+
| Markdown + BM25 | ✅ | ✅ |
|
|
164
|
+
| Self-host | ❌ | ✅ |
|
|
165
|
+
| Hosted search API | ✅ /search | ⚠️ DuckDuckGo HTML (no key) |
|
|
166
|
+
|
|
167
|
+
---
|
|
168
|
+
|
|
169
|
+
## 🔧 Development
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
git clone https://github.com/SanggonBoy/PyreCrawl.git
|
|
173
|
+
cd PyreCrawl
|
|
174
|
+
uv venv --python 3.12 .venv
|
|
175
|
+
source .venv/Scripts/activate # Windows; or .venv/bin/activate on macOS/Linux
|
|
176
|
+
uv pip install -e ".[dev]"
|
|
177
|
+
python -m playwright install chromium
|
|
178
|
+
scrapling install
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
### Run tests
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
python scripts/selfcheck.py # real-network smoke test
|
|
185
|
+
python scripts/probe_stdio.py # stdio JSON-RPC probe
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
---
|
|
189
|
+
|
|
190
|
+
## 📦 Publish
|
|
191
|
+
|
|
192
|
+
Maintainers only:
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
git tag v0.2.1
|
|
196
|
+
git push origin v0.2.1
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
GitHub Actions builds + uploads to PyPI via [trusted publishing](https://docs.pypi.org/trusted-publishers/).
|
|
200
|
+
|
|
201
|
+
---
|
|
202
|
+
|
|
203
|
+
## 📜 Uninstall
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
# Remove from all agent configs
|
|
207
|
+
pyrecrawl uninstall
|
|
208
|
+
|
|
209
|
+
# Remove the package
|
|
210
|
+
uv tool uninstall pyrecrawl
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
---
|
|
214
|
+
|
|
215
|
+
## 🛡️ License
|
|
216
|
+
|
|
217
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# 🔥 PyreCrawl — Self-Hosted Firecrawl Alternative as an MCP Server
|
|
2
|
+
|
|
3
|
+
[](https://opensource.org/licenses/MIT)
|
|
4
|
+
[](https://modelcontextprotocol.io/)
|
|
5
|
+
[](https://www.python.org/)
|
|
6
|
+
[](https://pypi.org/project/pyrecrawl/)
|
|
7
|
+
|
|
8
|
+
A **self-hosted Firecrawl alternative** combining two best-in-class open-source scrapers
|
|
9
|
+
behind a single Model Context Protocol (MCP) server:
|
|
10
|
+
|
|
11
|
+
- **[Scrapling](https://github.com/D4Vinci/Scrapling)** — fast HTTP (curl_cffi) + stealth browser with **Cloudflare Turnstile bypass**, adaptive element tracking, and XHR capture
|
|
12
|
+
- **[Crawl4AI](https://github.com/unclecode/crawl4ai)** — LLM-first crawler: BM25 fit-markdown, citations, structured extraction, deep crawl (BFS/DFS/BestFirst)
|
|
13
|
+
|
|
14
|
+
A **smart auto-fallback ladder** tries the cheapest engine that succeeds:
|
|
15
|
+
|
|
16
|
+
```
|
|
17
|
+
fast HTTP (Scrapling)
|
|
18
|
+
│ (403/503/Cloudflare challenge or empty body)
|
|
19
|
+
▼
|
|
20
|
+
stealth browser (Scrapling StealthyFetcher + CF solver)
|
|
21
|
+
│ (still blocked or page needs full JS rendering)
|
|
22
|
+
▼
|
|
23
|
+
full LLM processing (Crawl4AI AsyncWebCrawler + BM25)
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## ⚡ Tools exposed
|
|
27
|
+
|
|
28
|
+
| Tool | What it does |
|
|
29
|
+
|---|---|
|
|
30
|
+
| `scrape(url, prefer="auto")` | Single URL → LLM-ready markdown |
|
|
31
|
+
| `extract(url, schema)` | Scrape + structured extraction (JsonCss schema) |
|
|
32
|
+
| `map_site(root, include_pattern=None, limit=200)` | Enumerate all internal URLs |
|
|
33
|
+
| `crawl(root, max_pages=5, prefer="auto")` | Multi-page crawl with auto-fallback per page |
|
|
34
|
+
| `search(query, limit=10)` | Web search via DuckDuckGo HTML (no API key) |
|
|
35
|
+
| `health()` | Versions + import sanity check |
|
|
36
|
+
|
|
37
|
+
`prefer` options: `"auto"` (default ladder) · `"fast"` (HTTP only) · `"stealth"` (CF bypass) · `"llm"` (full Crawl4AI).
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
## 🚀 Install & Use (one-liner)
|
|
42
|
+
|
|
43
|
+
### 1. Install
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
# Using uv (recommended — fast, isolated, no venv needed)
|
|
47
|
+
uv tool install pyrecrawl
|
|
48
|
+
|
|
49
|
+
# Or pipx (alternative)
|
|
50
|
+
pipx install pyrecrawl
|
|
51
|
+
|
|
52
|
+
# Or pip into a venv
|
|
53
|
+
pip install pyrecrawl
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
### 2. One-time browser engines
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
pyrecrawl setup
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
This installs Playwright Chromium + Scrapling engines (~2 min, one-time).
|
|
63
|
+
|
|
64
|
+
### 3. Register with your AI agent
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
# Auto-detect installed agents and write their MCP configs
|
|
68
|
+
pyrecrawl install
|
|
69
|
+
|
|
70
|
+
# Or target specific agents
|
|
71
|
+
pyrecrawl install claude-desktop cursor
|
|
72
|
+
|
|
73
|
+
# Dry-run to preview what would change
|
|
74
|
+
pyrecrawl install --dry-run
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Supported agents: `claude-desktop`, `claude-code`, `cursor`, `vscode`, `codex`, `opencode`, `hermes`.
|
|
78
|
+
|
|
79
|
+
### 4. Start chatting
|
|
80
|
+
|
|
81
|
+
After installing + registering, **restart your agent** (or start a new session). Then ask:
|
|
82
|
+
|
|
83
|
+
> *"Scrape https://example.com and summarize it."*
|
|
84
|
+
|
|
85
|
+
The tools appear as `mcp_pyrecrawl_scrape`, `mcp_pyrecrawl_extract`, `mcp_pyrecrawl_map_site`, `mcp_pyrecrawl_crawl`, `mcp_pyrecrawl_search`, `mcp_pyrecrawl_health`.
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
89
|
+
## 📚 Manual config (if `pyrecrawl install` doesn't match your setup)
|
|
90
|
+
|
|
91
|
+
| Agent | Config file | Entry |
|
|
92
|
+
|---|---|---|
|
|
93
|
+
| **Claude Desktop** | `~/.config/Claude/claude_desktop_config.json` (Linux) / `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) / `%AppData%\Claude\claude_desktop_config.json` (Windows) | `{"mcpServers":{"pyrecrawl":{"command":"uvx","args":["--from","pyrecrawl","pyrecrawl","serve"]}}}` |
|
|
94
|
+
| **Claude Code** | project-scoped `.mcp.json` | same JSON as Claude Desktop |
|
|
95
|
+
| **Cursor** | `~/.cursor/mcp.json` | same JSON |
|
|
96
|
+
| **VS Code / Copilot** | `.vscode/mcp.json` (project) | `{"servers":{"pyrecrawl":{"command":"uvx","args":["--from","pyrecrawl","pyrecrawl","serve"],"type":"stdio"}}}` |
|
|
97
|
+
| **Codex CLI** | `~/.codex/config.toml` | `[mcp_servers.pyrecrawl]\ncommand = "uvx"\nargs = ["--from", "pyrecrawl", "pyrecrawl", "serve"]` |
|
|
98
|
+
| **OpenCode** | `~/.config/opencode/opencode.json` | `{"mcp":{"pyrecrawl":{"type":"local","command":["uvx","--from","pyrecrawl","pyrecrawl","serve"],"enabled":true}}}` |
|
|
99
|
+
| **Hermes** | `~/.hermes/config.yaml` or `%LocalAppData%\hermes\config.yaml` | `mcp_servers:\n pyrecrawl:\n command: uvx\n args:\n - --from\n - pyrecrawl\n - pyrecrawl\n - serve\n enabled: true` |
|
|
100
|
+
|
|
101
|
+
> **Windows note:** `uvx` must be on PATH. If not, use the full path to `uvx.exe` (e.g. `C:\Users\<you>\AppData\Local\hermes\bin\uvx.exe`).
|
|
102
|
+
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
## 🧠 Why two engines?
|
|
106
|
+
|
|
107
|
+
| Concern | Scrapling | Crawl4AI |
|
|
108
|
+
|---|---|---|
|
|
109
|
+
| Static HTML page | ✅ curl_cffi, ~200ms | ✅ browser overhead, ~3s |
|
|
110
|
+
| Cloudflare-protected | ✅ Turnstile solver | ⚠️ requires stealth setup |
|
|
111
|
+
| JS-heavy SPA | ✅ Chromium real browser | ✅ same |
|
|
112
|
+
| LLM-ready markdown | ⚠️ basic | ✅ BM25 + citations + fit |
|
|
113
|
+
| Structured extraction (CSS schema) | ❌ | ✅ JsonCss strategy |
|
|
114
|
+
| Deep crawl (BFS/DFS/BestFirst) | ✅ Spider + AutoThrottle | ✅ BFS/DFS/BestFirst + adaptive |
|
|
115
|
+
| Adaptive element tracking | ✅ parser relocates moved elements | ❌ |
|
|
116
|
+
|
|
117
|
+
**PyreCrawl = Scrapling for fetch & bypass + Crawl4AI for processing & extraction.**
|
|
118
|
+
|
|
119
|
+
---
|
|
120
|
+
|
|
121
|
+
## 📊 Compared to Firecrawl (hosted)
|
|
122
|
+
|
|
123
|
+
| | Firecrawl | PyreCrawl |
|
|
124
|
+
|---|---|---|
|
|
125
|
+
| Cost | Free 1k/mo, then $16–333/mo | **Free, self-hosted** |
|
|
126
|
+
| Local LLM support | ❌ | ✅ Ollama / any LLM |
|
|
127
|
+
| Cloudflare bypass | ✅ (Fire-Engine, paid) | ✅ (free, Scrapling) |
|
|
128
|
+
| Markdown + BM25 | ✅ | ✅ |
|
|
129
|
+
| Self-host | ❌ | ✅ |
|
|
130
|
+
| Hosted search API | ✅ /search | ⚠️ DuckDuckGo HTML (no key) |
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
## 🔧 Development
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
git clone https://github.com/SanggonBoy/PyreCrawl.git
|
|
138
|
+
cd PyreCrawl
|
|
139
|
+
uv venv --python 3.12 .venv
|
|
140
|
+
source .venv/Scripts/activate # Windows; or .venv/bin/activate on macOS/Linux
|
|
141
|
+
uv pip install -e ".[dev]"
|
|
142
|
+
python -m playwright install chromium
|
|
143
|
+
scrapling install
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
### Run tests
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
python scripts/selfcheck.py # real-network smoke test
|
|
150
|
+
python scripts/probe_stdio.py # stdio JSON-RPC probe
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
---
|
|
154
|
+
|
|
155
|
+
## 📦 Publish
|
|
156
|
+
|
|
157
|
+
Maintainers only:
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
git tag v0.2.1
|
|
161
|
+
git push origin v0.2.1
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
GitHub Actions builds + uploads to PyPI via [trusted publishing](https://docs.pypi.org/trusted-publishers/).
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
## 📜 Uninstall
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
# Remove from all agent configs
|
|
172
|
+
pyrecrawl uninstall
|
|
173
|
+
|
|
174
|
+
# Remove the package
|
|
175
|
+
uv tool uninstall pyrecrawl
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
---
|
|
179
|
+
|
|
180
|
+
## 🛡️ License
|
|
181
|
+
|
|
182
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "pyrecrawl"
|
|
3
|
+
version = "0.2.1"
|
|
4
|
+
description = "Self-hosted Firecrawl alternative: LLM-ready scraper/crawler MCP server combining Crawl4AI and Scrapling with a smart auto-fallback ladder."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = { text = "MIT" }
|
|
7
|
+
authors = [
|
|
8
|
+
{ name = "Alexander Jay" },
|
|
9
|
+
]
|
|
10
|
+
keywords = ["mcp", "scraping", "crawl4ai", "scrapling", "firecrawl-alternative", "llm", "rag", "ai-agent", "web-scraping", "ai-agent-tools"]
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Development Status :: 4 - Beta",
|
|
14
|
+
"Intended Audience :: Developers",
|
|
15
|
+
"License :: OSI Approved :: MIT License",
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Programming Language :: Python :: 3.10",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
|
21
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
22
|
+
"Topic :: Text Processing :: Markup :: Markdown",
|
|
23
|
+
]
|
|
24
|
+
dependencies = [
|
|
25
|
+
"crawl4ai>=0.9.0",
|
|
26
|
+
"scrapling[fetchers]>=0.4.0",
|
|
27
|
+
"mcp>=1.0,<2.0",
|
|
28
|
+
"pydantic>=2.0",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
dev = [
|
|
33
|
+
"pytest>=8.0",
|
|
34
|
+
"pytest-asyncio>=0.23",
|
|
35
|
+
]
|
|
36
|
+
browsers = [
|
|
37
|
+
"playwright>=1.40",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[project.scripts]
|
|
41
|
+
pyrecrawl = "pyrecrawl.cli:main"
|
|
42
|
+
|
|
43
|
+
[project.urls]
|
|
44
|
+
Homepage = "https://github.com/SanggonBoy/PyreCrawl"
|
|
45
|
+
Repository = "https://github.com/SanggonBoy/PyreCrawl"
|
|
46
|
+
Documentation = "https://github.com/SanggonBoy/PyreCrawl#readme"
|
|
47
|
+
Issues = "https://github.com/SanggonBoy/PyreCrawl/issues"
|
|
48
|
+
Changelog = "https://github.com/SanggonBoy/PyreCrawl/releases"
|
|
49
|
+
|
|
50
|
+
[build-system]
|
|
51
|
+
requires = ["hatchling"]
|
|
52
|
+
build-backend = "hatchling.build"
|
|
53
|
+
|
|
54
|
+
[tool.hatch.build.targets.wheel]
|
|
55
|
+
packages = ["src/pyrecrawl"]
|
|
56
|
+
|
|
57
|
+
[tool.pytest.ini_options]
|
|
58
|
+
asyncio_mode = "auto"
|
|
59
|
+
# no tests/ dir yet — executable checks live in scripts/selfcheck.py + scripts/probe_stdio.py
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Probe the MCP server over stdio: verifies JSON-RPC stays clean on stdout while invoking real tools."""
|
|
2
|
+
import json
|
|
3
|
+
import subprocess
|
|
4
|
+
import sys
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
10
|
+
VENV_PY = ROOT / ".venv" / "Scripts" / "python.exe"
|
|
11
|
+
|
|
12
|
+
proc = subprocess.Popen(
|
|
13
|
+
[str(VENV_PY), "-m", "pyrecrawl.server"],
|
|
14
|
+
stdin=subprocess.PIPE,
|
|
15
|
+
stdout=subprocess.PIPE,
|
|
16
|
+
stderr=subprocess.PIPE,
|
|
17
|
+
text=True,
|
|
18
|
+
bufsize=1,
|
|
19
|
+
cwd=ROOT,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
stderr_lines: list[str] = []
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _drain_stderr():
|
|
26
|
+
for line in proc.stderr:
|
|
27
|
+
stderr_lines.append(line)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
threading.Thread(target=_drain_stderr, daemon=True).start()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def rpc(method, params=None, nid=0, timeout=120):
|
|
34
|
+
msg = {"jsonrpc": "2.0", "id": nid, "method": method, "params": params or {}}
|
|
35
|
+
proc.stdin.write(json.dumps(msg) + "\n")
|
|
36
|
+
proc.stdin.flush()
|
|
37
|
+
deadline = time.time() + timeout
|
|
38
|
+
while time.time() < deadline:
|
|
39
|
+
line = proc.stdout.readline()
|
|
40
|
+
if not line:
|
|
41
|
+
if proc.poll() is not None:
|
|
42
|
+
raise RuntimeError(f"server died; stderr tail: {''.join(stderr_lines[-15:])}")
|
|
43
|
+
continue
|
|
44
|
+
line = line.strip()
|
|
45
|
+
if not line:
|
|
46
|
+
continue
|
|
47
|
+
return json.loads(line)
|
|
48
|
+
raise TimeoutError(f"no reply for {method} after {timeout}s")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def payload_of(out):
|
|
52
|
+
return json.loads(out["result"]["content"][0]["text"])
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def main():
|
|
56
|
+
rid = 1
|
|
57
|
+
print("== initialize ==")
|
|
58
|
+
out = rpc("initialize", {
|
|
59
|
+
"protocolVersion": "2024-11-05",
|
|
60
|
+
"capabilities": {},
|
|
61
|
+
"clientInfo": {"name": "probe", "version": "0.1"},
|
|
62
|
+
}, rid); rid += 1
|
|
63
|
+
assert "result" in out, out
|
|
64
|
+
print(" server:", out["result"]["serverInfo"])
|
|
65
|
+
proc.stdin.write(json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized"}) + "\n")
|
|
66
|
+
proc.stdin.flush()
|
|
67
|
+
|
|
68
|
+
print("== tools/list ==")
|
|
69
|
+
out = rpc("tools/list", {}, rid); rid += 1
|
|
70
|
+
names = [t["name"] for t in out["result"]["tools"]]
|
|
71
|
+
print(" tools:", names)
|
|
72
|
+
assert set(names) == {"scrape", "extract", "map_site", "crawl", "search", "health"}
|
|
73
|
+
|
|
74
|
+
print("== health ==")
|
|
75
|
+
p = payload_of(rpc("tools/call", {"name": "health", "arguments": {}}, rid)); rid += 1
|
|
76
|
+
print(" versions:", {k: v for k, v in p.items() if not k.endswith("error")})
|
|
77
|
+
assert "crawl4ai" in p and "scrapling" in p, p
|
|
78
|
+
|
|
79
|
+
print("== scrape example.com (fast) ==")
|
|
80
|
+
p = payload_of(rpc("tools/call", {
|
|
81
|
+
"name": "scrape",
|
|
82
|
+
"arguments": {"url": "https://example.com", "prefer": "fast", "timeout": 20},
|
|
83
|
+
}, rid)); rid += 1
|
|
84
|
+
print(" status:", p.get("status"), "| title:", p.get("title"), "| method:", p.get("method"))
|
|
85
|
+
assert p.get("status") == 200 and p.get("title") == "Example Domain", p
|
|
86
|
+
|
|
87
|
+
print("== extract (CSS schema) ==")
|
|
88
|
+
p = payload_of(rpc("tools/call", {
|
|
89
|
+
"name": "extract",
|
|
90
|
+
"arguments": {
|
|
91
|
+
"url": "https://example.com",
|
|
92
|
+
"schema": {"name": "Page", "baseSelector": "body",
|
|
93
|
+
"fields": [{"name": "heading", "selector": "h1", "type": "text"}]},
|
|
94
|
+
},
|
|
95
|
+
}, rid)); rid += 1
|
|
96
|
+
print(" data:", p.get("data"), "| method:", p.get("method"))
|
|
97
|
+
assert "error" not in p, p
|
|
98
|
+
|
|
99
|
+
print("== search (ladder: fast -> stealth) ==")
|
|
100
|
+
p = payload_of(rpc("tools/call", {
|
|
101
|
+
"name": "search",
|
|
102
|
+
"arguments": {"query": "firecrawl alternative open source", "limit": 3},
|
|
103
|
+
}, rid, timeout=180)); rid += 1
|
|
104
|
+
print(" count:", p.get("count"))
|
|
105
|
+
for x in (p.get("results") or [])[:3]:
|
|
106
|
+
print(" -", (x.get("title") or "")[:60], "|", (x.get("url") or "")[:60])
|
|
107
|
+
assert p.get("count", 0) >= 1, p
|
|
108
|
+
|
|
109
|
+
print("\nALL PROBE CHECKS PASSED — stdout is pure JSON-RPC")
|
|
110
|
+
proc.terminate()
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
if __name__ == "__main__":
|
|
114
|
+
sys.exit(main())
|