deepcloak 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deepcloak-0.1.0/.gitignore +30 -0
- deepcloak-0.1.0/CHANGELOG.md +29 -0
- deepcloak-0.1.0/LICENSE +21 -0
- deepcloak-0.1.0/NOTICE +21 -0
- deepcloak-0.1.0/PKG-INFO +157 -0
- deepcloak-0.1.0/README.md +134 -0
- deepcloak-0.1.0/pyproject.toml +51 -0
- deepcloak-0.1.0/showcase/sample/README.md +14 -0
- deepcloak-0.1.0/src/deepcloak/__init__.py +3 -0
- deepcloak-0.1.0/src/deepcloak/bot_wall_detector.py +68 -0
- deepcloak-0.1.0/src/deepcloak/cli.py +87 -0
- deepcloak-0.1.0/src/deepcloak/config.py +168 -0
- deepcloak-0.1.0/src/deepcloak/evidence.py +67 -0
- deepcloak-0.1.0/src/deepcloak/fetch_router.py +109 -0
- deepcloak-0.1.0/src/deepcloak/ldr_shim.py +115 -0
- deepcloak-0.1.0/src/deepcloak/mcp_server.py +54 -0
- deepcloak-0.1.0/src/deepcloak/progress.py +36 -0
- deepcloak-0.1.0/src/deepcloak/research_core.py +141 -0
- deepcloak-0.1.0/src/deepcloak/retriever.py +118 -0
- deepcloak-0.1.0/src/deepcloak/setup.py +77 -0
- deepcloak-0.1.0/src/deepcloak/stealth_downloader.py +47 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
dist/
|
|
7
|
+
build/
|
|
8
|
+
.venv/
|
|
9
|
+
venv/
|
|
10
|
+
.pytest_cache/
|
|
11
|
+
.ruff_cache/
|
|
12
|
+
.mypy_cache/
|
|
13
|
+
|
|
14
|
+
# Env / secrets
|
|
15
|
+
.env
|
|
16
|
+
.env.*
|
|
17
|
+
CLAUDE.local.md
|
|
18
|
+
|
|
19
|
+
# Showcase (Next.js)
|
|
20
|
+
showcase/node_modules/
|
|
21
|
+
showcase/.next/
|
|
22
|
+
showcase/out/
|
|
23
|
+
showcase/next-env.d.ts
|
|
24
|
+
showcase/*.tsbuildinfo
|
|
25
|
+
|
|
26
|
+
# Local working notes (issues live on GitHub)
|
|
27
|
+
.scratch/
|
|
28
|
+
|
|
29
|
+
# OS
|
|
30
|
+
.DS_Store
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to DeepCloak are documented here. Format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/); this project uses [SemVer](https://semver.org/).
|
|
5
|
+
|
|
6
|
+
## [Unreleased]
|
|
7
|
+
|
|
8
|
+
## [0.1.0] — 2026-06-05
|
|
9
|
+
|
|
10
|
+
First public release.
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- **Deep Research that reads Bot-walled pages.** Plain fetch first; on a Bot Wall
|
|
14
|
+
(Cloudflare / Datadome / Turnstile / reCAPTCHA) it Escalates one URL to a Stealth Fetch
|
|
15
|
+
and Bypasses it — recovering content other agents drop.
|
|
16
|
+
- **StealthRetriever** — handed to `local-deep-research` via its `retrievers=` API so the
|
|
17
|
+
research loop synthesises over Bypassed full pages, not snippets.
|
|
18
|
+
- **Evidence Records** — every fetch is recorded (Bot Wall kind, Escalation, Bypass,
|
|
19
|
+
plain status, timing); reports end with a `🛡️ Bypassed N bot-walled sources` badge and a
|
|
20
|
+
`*.evidence.json` sidecar.
|
|
21
|
+
- Three surfaces over one core: **CLI**, **MCP server** (`deep_research` / `quick_summary`
|
|
22
|
+
/ `get_evidence`), and a **Claude skill**.
|
|
23
|
+
- Local-first: works with `ollama` or any OpenAI-compatible local endpoint — no API key
|
|
24
|
+
required. Default search is DuckDuckGo (keyless); SearXNG is opt-in.
|
|
25
|
+
- `--respect-robots` to honor robots.txt (ignored by default — see ADR-0002).
|
|
26
|
+
- Per-document content cap so small-context local models can finish a report.
|
|
27
|
+
|
|
28
|
+
[Unreleased]: https://github.com/Mrbaeksang/deepcloak/compare/v0.1.0...HEAD
|
|
29
|
+
[0.1.0]: https://github.com/Mrbaeksang/deepcloak/releases/tag/v0.1.0
|
deepcloak-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mrbaeksang
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
deepcloak-0.1.0/NOTICE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
DeepCloak
|
|
2
|
+
Copyright (c) 2026 Mrbaeksang
|
|
3
|
+
|
|
4
|
+
This product depends on, and is built around, the following open-source projects.
|
|
5
|
+
DeepCloak does not vendor their code; it depends on them via pip.
|
|
6
|
+
|
|
7
|
+
-------------------------------------------------------------------------------
|
|
8
|
+
local-deep-research
|
|
9
|
+
Homepage: https://github.com/LearningCircuit/local-deep-research
|
|
10
|
+
License: MIT
|
|
11
|
+
Role: the iterative multi-source Deep Research loop.
|
|
12
|
+
|
|
13
|
+
-------------------------------------------------------------------------------
|
|
14
|
+
CloakBrowser
|
|
15
|
+
Homepage: https://github.com/CloakHQ/CloakBrowser
|
|
16
|
+
License: MIT
|
|
17
|
+
Role: the stealth Chromium that performs the Stealth Fetch / Bypass.
|
|
18
|
+
|
|
19
|
+
-------------------------------------------------------------------------------
|
|
20
|
+
Both are MIT-licensed; their copyright and permission notices apply to their
|
|
21
|
+
respective code. See each project's repository for full license text.
|
deepcloak-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: deepcloak
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Local-first deep research agent that reads the whole web — even pages behind Cloudflare, Datadome, Turnstile & reCAPTCHA.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Mrbaeksang/deepcloak
|
|
6
|
+
Project-URL: Repository, https://github.com/Mrbaeksang/deepcloak
|
|
7
|
+
Project-URL: Author, https://baeksang.dev
|
|
8
|
+
Author-email: Mrbaeksang <contact@baeksang.dev>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
License-File: NOTICE
|
|
12
|
+
Keywords: ai-agent,anti-bot,cloudflare,deep-research,llm,local-first,mcp,rag,stealth,web-scraping
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Requires-Dist: cloakbrowser>=0.3.27
|
|
15
|
+
Requires-Dist: local-deep-research==1.6.11
|
|
16
|
+
Requires-Dist: requests>=2.31
|
|
17
|
+
Provides-Extra: dev
|
|
18
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
19
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
20
|
+
Provides-Extra: mcp
|
|
21
|
+
Requires-Dist: mcp>=1.0; extra == 'mcp'
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
<div align="center">
|
|
25
|
+
|
|
26
|
+
<h1>🛡️ DeepCloak</h1>
|
|
27
|
+
|
|
28
|
+
### The deep-research agent that reads the pages others can't.
|
|
29
|
+
|
|
30
|
+
**Cloudflare · Datadome · Turnstile · reCAPTCHA — it walks straight through them.**
|
|
31
|
+
|
|
32
|
+
[](https://github.com/Mrbaeksang/deepcloak/actions/workflows/ci.yml)
|
|
33
|
+
[](LICENSE)
|
|
34
|
+
[](pyproject.toml)
|
|
35
|
+
[](#-use-it-from-an-ai-agent)
|
|
36
|
+
[](CONTRIBUTING.md)
|
|
37
|
+
[](https://github.com/Mrbaeksang/deepcloak/stargazers)
|
|
38
|
+
[](https://youtu.be/p5ompjDLzaI)
|
|
39
|
+
|
|
40
|
+
**English** · [한국어](README.ko.md) · [简体中文](README.zh-CN.md)
|
|
41
|
+
|
|
42
|
+
### [▶ Live demo — deepcloak.vercel.app](https://deepcloak.vercel.app) · [▶ Watch on YouTube](https://youtu.be/p5ompjDLzaI)
|
|
43
|
+
|
|
44
|
+
[](https://youtu.be/p5ompjDLzaI)
|
|
45
|
+
|
|
46
|
+
</div>
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
## The problem
|
|
51
|
+
|
|
52
|
+
You ask a research tool a question. Half the best sources sit behind a **Bot Wall** — Cloudflare, Datadome, a Turnstile, a reCAPTCHA. Every other tool gets a `403`, silently drops those pages, and hands you a thinner report. **You never even learn what it missed.**
|
|
53
|
+
|
|
54
|
+
## What DeepCloak does
|
|
55
|
+
|
|
56
|
+
When a plain fetch hits a Bot Wall, DeepCloak **Escalates** that one URL to a **Stealth Fetch** and **Bypasses** the wall — recovering the content other agents abandon. Then it tells you, at the bottom of every report, exactly how many walls it broke through.
|
|
57
|
+
|
|
58
|
+
It's a thin, local-first orchestrator over two great projects: [`local-deep-research`](https://github.com/LearningCircuit/local-deep-research) (the research loop) and [`CloakBrowser`](https://github.com/CloakHQ/CloakBrowser) (the stealth browser). Use it as a **CLI**, an **MCP server**, or a **Claude skill**. MIT.
|
|
59
|
+
|
|
60
|
+
## 🌑 Why we built this
|
|
61
|
+
|
|
62
|
+
The open web is quietly closing. More of the best writing now sits behind a bot check, and AI research agents — the tools we increasingly trust to read the web *for us* — go blind at exactly those doors, without ever saying so. A report that silently skips every walled source isn't neutral; it's wrong in a way you can't see.
|
|
63
|
+
|
|
64
|
+
DeepCloak's stance is simple: **your agent should be able to read what a person with a browser can read** — and it should be honest about how it got there. So it Bypasses the wall when it has to, keeps everything local (no query or page leaves your machine), and prints an Evidence Record of every wall it crossed. Capability *and* transparency, MIT-licensed, no lock-in.
|
|
65
|
+
|
|
66
|
+
## ✨ Why it's different
|
|
67
|
+
|
|
68
|
+
| | Plain deep research | **DeepCloak** |
|
|
69
|
+
| --- | :---: | :---: |
|
|
70
|
+
| Reads the open web | ✅ | ✅ |
|
|
71
|
+
| Reads Cloudflare / Datadome / Turnstile / reCAPTCHA pages | ❌ *dropped silently* | ✅ **Bypassed** |
|
|
72
|
+
| Tells you which sources were walled | ❌ | ✅ Evidence Record |
|
|
73
|
+
| Local-first (no API key required) | ✅ | ✅ |
|
|
74
|
+
| Fast on open pages | — | ✅ *plain-first, stealth only when needed* |
|
|
75
|
+
|
|
76
|
+
> **Verified live — not mocked.** The clip above is an **unedited screen recording** (captured with `ffmpeg`, no compositing) of a real `deepcloak` run against a **local LLM (Qwen) + SearXNG — no API key**. It Escalates on each Bot Wall and **Bypasses 8 Cloudflare/Turnstile walls** in one pass, then writes a cited report. Full clip: [`docs/media/demo-real.mp4`](docs/media/demo-real.mp4); a raw asciinema session is also kept at [`docs/media/demo.cast`](docs/media/demo.cast). Wall counts vary per run (8–20) because the open web does.
|
|
77
|
+
|
|
78
|
+
## 🚀 Quickstart
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
pip install deepcloak
|
|
82
|
+
deepcloak setup # one-time: downloads the stealth browser
|
|
83
|
+
export OPENAI_API_KEY=... # or ANTHROPIC_API_KEY / GEMINI_API_KEY — or --provider ollama
|
|
84
|
+
deepcloak "How does Cloudflare Turnstile detect bots?" --depth detailed --out report.md
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
You get a cited `report.md` ending with a `🛡️ Bypassed N bot-walled sources` section, plus a `report.md.evidence.json` sidecar.
|
|
88
|
+
|
|
89
|
+
## 🧠 How it works
|
|
90
|
+
|
|
91
|
+
```
|
|
92
|
+
search (DuckDuckGo, no setup) ─▶ candidate URLs
|
|
93
|
+
│
|
|
94
|
+
▼ for each page:
|
|
95
|
+
plain fetch ─▶ Bot Wall detected? ──no──▶ use it (fast)
|
|
96
|
+
│ yes
|
|
97
|
+
▼
|
|
98
|
+
Escalate ─▶ Stealth Fetch (CloakBrowser) ─▶ Bypass
|
|
99
|
+
│
|
|
100
|
+
▼
|
|
101
|
+
research loop (local-deep-research) ─▶ cited report + Evidence Records
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Stealth is heavy, so DeepCloak tries a cheap plain fetch first and only launches the stealth browser when it actually detects a Bot Wall (`--stealth auto`, the default). Use `--depth detailed`/`report` to fetch full pages where Bypasses happen.
|
|
105
|
+
|
|
106
|
+
## 🤖 Connect it to your agent (MCP)
|
|
107
|
+
|
|
108
|
+
DeepCloak runs as a stdio **MCP server** exposing `deep_research(query, depth)`, `quick_summary(query)`, and `get_evidence(run_id)`.
|
|
109
|
+
|
|
110
|
+
**Claude Code** — add to your project's `.mcp.json` (an example ships in this repo):
|
|
111
|
+
|
|
112
|
+
```json
|
|
113
|
+
{ "mcpServers": { "deepcloak": { "command": "deepcloak", "args": ["mcp"] } } }
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
**Codex** — add to `~/.codex/config.toml`:
|
|
117
|
+
|
|
118
|
+
```toml
|
|
119
|
+
[mcp_servers.deepcloak]
|
|
120
|
+
command = "deepcloak"
|
|
121
|
+
args = ["mcp"]
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Then your agent can call `deep_research` and read bot-walled sources directly. Prefer a slash-style skill? Drop [`skill/SKILL.md`](skill/SKILL.md) into `~/.claude/skills/deepcloak/`.
|
|
125
|
+
|
|
126
|
+
## ⚙️ Configuration
|
|
127
|
+
|
|
128
|
+
| Flag | Default | Notes |
|
|
129
|
+
| --- | --- | --- |
|
|
130
|
+
| `--depth` | `detailed` | `quick` / `detailed` / `report` |
|
|
131
|
+
| `--engine` | `duckduckgo` | `searxng` / `auto` |
|
|
132
|
+
| `--stealth` | `auto` | `always` / `off` |
|
|
133
|
+
| `--provider` / `--model` | auto-detected | `OPENAI` → `ANTHROPIC` → `GEMINI`, or `ollama` |
|
|
134
|
+
| `--respect-robots` | off | honor robots.txt |
|
|
135
|
+
| `--proxy` | — | SOCKS5 for the Stealth Fetch |
|
|
136
|
+
|
|
137
|
+
## ⚠️ Responsible use
|
|
138
|
+
|
|
139
|
+
DeepCloak Bypasses bot-detection. **You are responsible for having the right to access whatever you fetch.** robots.txt is **ignored by default**; pass `--respect-robots` to honor it ([ADR-0002](docs/adr/0002-ignore-robots-by-default.md)). Don't use it to violate sites' terms or the law.
|
|
140
|
+
|
|
141
|
+
## 🛠️ Built on
|
|
142
|
+
|
|
143
|
+
[`local-deep-research`](https://github.com/LearningCircuit/local-deep-research) (MIT) + [`CloakBrowser`](https://github.com/CloakHQ/CloakBrowser) (MIT), via pip — no vendored code. Domain glossary in [CONTEXT.md](CONTEXT.md); design decisions in [docs/adr/](docs/adr/); contributing guide in [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
144
|
+
|
|
145
|
+
## 📄 License
|
|
146
|
+
|
|
147
|
+
MIT — see [LICENSE](LICENSE) and [NOTICE](NOTICE).
|
|
148
|
+
|
|
149
|
+
<div align="center">
|
|
150
|
+
|
|
151
|
+
**If DeepCloak read a page your last tool gave up on, drop a ⭐ — it helps others find it.**
|
|
152
|
+
|
|
153
|
+
Built by **[Mrbaeksang](https://baeksang.dev)** · [baeksang.dev](https://baeksang.dev) · [contact@baeksang.dev](mailto:contact@baeksang.dev)
|
|
154
|
+
|
|
155
|
+
[](https://star-history.com/#Mrbaeksang/deepcloak&Date)
|
|
156
|
+
|
|
157
|
+
</div>
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
<h1>🛡️ DeepCloak</h1>
|
|
4
|
+
|
|
5
|
+
### The deep-research agent that reads the pages others can't.
|
|
6
|
+
|
|
7
|
+
**Cloudflare · Datadome · Turnstile · reCAPTCHA — it walks straight through them.**
|
|
8
|
+
|
|
9
|
+
[](https://github.com/Mrbaeksang/deepcloak/actions/workflows/ci.yml)
|
|
10
|
+
[](LICENSE)
|
|
11
|
+
[](pyproject.toml)
|
|
12
|
+
[](#-use-it-from-an-ai-agent)
|
|
13
|
+
[](CONTRIBUTING.md)
|
|
14
|
+
[](https://github.com/Mrbaeksang/deepcloak/stargazers)
|
|
15
|
+
[](https://youtu.be/p5ompjDLzaI)
|
|
16
|
+
|
|
17
|
+
**English** · [한국어](README.ko.md) · [简体中文](README.zh-CN.md)
|
|
18
|
+
|
|
19
|
+
### [▶ Live demo — deepcloak.vercel.app](https://deepcloak.vercel.app) · [▶ Watch on YouTube](https://youtu.be/p5ompjDLzaI)
|
|
20
|
+
|
|
21
|
+
[](https://youtu.be/p5ompjDLzaI)
|
|
22
|
+
|
|
23
|
+
</div>
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
## The problem
|
|
28
|
+
|
|
29
|
+
You ask a research tool a question. Half the best sources sit behind a **Bot Wall** — Cloudflare, Datadome, a Turnstile, a reCAPTCHA. Every other tool gets a `403`, silently drops those pages, and hands you a thinner report. **You never even learn what it missed.**
|
|
30
|
+
|
|
31
|
+
## What DeepCloak does
|
|
32
|
+
|
|
33
|
+
When a plain fetch hits a Bot Wall, DeepCloak **Escalates** that one URL to a **Stealth Fetch** and **Bypasses** the wall — recovering the content other agents abandon. Then it tells you, at the bottom of every report, exactly how many walls it broke through.
|
|
34
|
+
|
|
35
|
+
It's a thin, local-first orchestrator over two great projects: [`local-deep-research`](https://github.com/LearningCircuit/local-deep-research) (the research loop) and [`CloakBrowser`](https://github.com/CloakHQ/CloakBrowser) (the stealth browser). Use it as a **CLI**, an **MCP server**, or a **Claude skill**. MIT.
|
|
36
|
+
|
|
37
|
+
## 🌑 Why we built this
|
|
38
|
+
|
|
39
|
+
The open web is quietly closing. More of the best writing now sits behind a bot check, and AI research agents — the tools we increasingly trust to read the web *for us* — go blind at exactly those doors, without ever saying so. A report that silently skips every walled source isn't neutral; it's wrong in a way you can't see.
|
|
40
|
+
|
|
41
|
+
DeepCloak's stance is simple: **your agent should be able to read what a person with a browser can read** — and it should be honest about how it got there. So it Bypasses the wall when it has to, keeps everything local (no query or page leaves your machine), and prints an Evidence Record of every wall it crossed. Capability *and* transparency, MIT-licensed, no lock-in.
|
|
42
|
+
|
|
43
|
+
## ✨ Why it's different
|
|
44
|
+
|
|
45
|
+
| | Plain deep research | **DeepCloak** |
|
|
46
|
+
| --- | :---: | :---: |
|
|
47
|
+
| Reads the open web | ✅ | ✅ |
|
|
48
|
+
| Reads Cloudflare / Datadome / Turnstile / reCAPTCHA pages | ❌ *dropped silently* | ✅ **Bypassed** |
|
|
49
|
+
| Tells you which sources were walled | ❌ | ✅ Evidence Record |
|
|
50
|
+
| Local-first (no API key required) | ✅ | ✅ |
|
|
51
|
+
| Fast on open pages | — | ✅ *plain-first, stealth only when needed* |
|
|
52
|
+
|
|
53
|
+
> **Verified live — not mocked.** The clip above is an **unedited screen recording** (captured with `ffmpeg`, no compositing) of a real `deepcloak` run against a **local LLM (Qwen) + SearXNG — no API key**. It Escalates on each Bot Wall and **Bypasses 8 Cloudflare/Turnstile walls** in one pass, then writes a cited report. Full clip: [`docs/media/demo-real.mp4`](docs/media/demo-real.mp4); a raw asciinema session is also kept at [`docs/media/demo.cast`](docs/media/demo.cast). Wall counts vary per run (8–20) because the open web does.
|
|
54
|
+
|
|
55
|
+
## 🚀 Quickstart
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install deepcloak
|
|
59
|
+
deepcloak setup # one-time: downloads the stealth browser
|
|
60
|
+
export OPENAI_API_KEY=... # or ANTHROPIC_API_KEY / GEMINI_API_KEY — or --provider ollama
|
|
61
|
+
deepcloak "How does Cloudflare Turnstile detect bots?" --depth detailed --out report.md
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
You get a cited `report.md` ending with a `🛡️ Bypassed N bot-walled sources` section, plus a `report.md.evidence.json` sidecar.
|
|
65
|
+
|
|
66
|
+
## 🧠 How it works
|
|
67
|
+
|
|
68
|
+
```
|
|
69
|
+
search (DuckDuckGo, no setup) ─▶ candidate URLs
|
|
70
|
+
│
|
|
71
|
+
▼ for each page:
|
|
72
|
+
plain fetch ─▶ Bot Wall detected? ──no──▶ use it (fast)
|
|
73
|
+
│ yes
|
|
74
|
+
▼
|
|
75
|
+
Escalate ─▶ Stealth Fetch (CloakBrowser) ─▶ Bypass
|
|
76
|
+
│
|
|
77
|
+
▼
|
|
78
|
+
research loop (local-deep-research) ─▶ cited report + Evidence Records
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Stealth is heavy, so DeepCloak tries a cheap plain fetch first and only launches the stealth browser when it actually detects a Bot Wall (`--stealth auto`, the default). Use `--depth detailed`/`report` to fetch full pages where Bypasses happen.
|
|
82
|
+
|
|
83
|
+
## 🤖 Connect it to your agent (MCP)
|
|
84
|
+
|
|
85
|
+
DeepCloak runs as a stdio **MCP server** exposing `deep_research(query, depth)`, `quick_summary(query)`, and `get_evidence(run_id)`.
|
|
86
|
+
|
|
87
|
+
**Claude Code** — add to your project's `.mcp.json` (an example ships in this repo):
|
|
88
|
+
|
|
89
|
+
```json
|
|
90
|
+
{ "mcpServers": { "deepcloak": { "command": "deepcloak", "args": ["mcp"] } } }
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
**Codex** — add to `~/.codex/config.toml`:
|
|
94
|
+
|
|
95
|
+
```toml
|
|
96
|
+
[mcp_servers.deepcloak]
|
|
97
|
+
command = "deepcloak"
|
|
98
|
+
args = ["mcp"]
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Then your agent can call `deep_research` and read bot-walled sources directly. Prefer a slash-style skill? Drop [`skill/SKILL.md`](skill/SKILL.md) into `~/.claude/skills/deepcloak/`.
|
|
102
|
+
|
|
103
|
+
## ⚙️ Configuration
|
|
104
|
+
|
|
105
|
+
| Flag | Default | Notes |
|
|
106
|
+
| --- | --- | --- |
|
|
107
|
+
| `--depth` | `detailed` | `quick` / `detailed` / `report` |
|
|
108
|
+
| `--engine` | `duckduckgo` | `searxng` / `auto` |
|
|
109
|
+
| `--stealth` | `auto` | `always` / `off` |
|
|
110
|
+
| `--provider` / `--model` | auto-detected | `OPENAI` → `ANTHROPIC` → `GEMINI`, or `ollama` |
|
|
111
|
+
| `--respect-robots` | off | honor robots.txt |
|
|
112
|
+
| `--proxy` | — | SOCKS5 for the Stealth Fetch |
|
|
113
|
+
|
|
114
|
+
## ⚠️ Responsible use
|
|
115
|
+
|
|
116
|
+
DeepCloak Bypasses bot-detection. **You are responsible for having the right to access whatever you fetch.** robots.txt is **ignored by default**; pass `--respect-robots` to honor it ([ADR-0002](docs/adr/0002-ignore-robots-by-default.md)). Don't use it to violate sites' terms or the law.
|
|
117
|
+
|
|
118
|
+
## 🛠️ Built on
|
|
119
|
+
|
|
120
|
+
[`local-deep-research`](https://github.com/LearningCircuit/local-deep-research) (MIT) + [`CloakBrowser`](https://github.com/CloakHQ/CloakBrowser) (MIT), via pip — no vendored code. Domain glossary in [CONTEXT.md](CONTEXT.md); design decisions in [docs/adr/](docs/adr/); contributing guide in [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
121
|
+
|
|
122
|
+
## 📄 License
|
|
123
|
+
|
|
124
|
+
MIT — see [LICENSE](LICENSE) and [NOTICE](NOTICE).
|
|
125
|
+
|
|
126
|
+
<div align="center">
|
|
127
|
+
|
|
128
|
+
**If DeepCloak read a page your last tool gave up on, drop a ⭐ — it helps others find it.**
|
|
129
|
+
|
|
130
|
+
Built by **[Mrbaeksang](https://baeksang.dev)** · [baeksang.dev](https://baeksang.dev) · [contact@baeksang.dev](mailto:contact@baeksang.dev)
|
|
131
|
+
|
|
132
|
+
[](https://star-history.com/#Mrbaeksang/deepcloak&Date)
|
|
133
|
+
|
|
134
|
+
</div>
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "deepcloak"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Local-first deep research agent that reads the whole web — even pages behind Cloudflare, Datadome, Turnstile & reCAPTCHA."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
authors = [{ name = "Mrbaeksang", email = "contact@baeksang.dev" }]
|
|
13
|
+
keywords = ["deep-research", "web-scraping", "stealth", "anti-bot", "cloudflare", "mcp", "llm", "ai-agent", "rag", "local-first"]
|
|
14
|
+
|
|
15
|
+
# ADR-0001: local-deep-research is pinned with == because the stealth shim
|
|
16
|
+
# depends on its internal symbols.
|
|
17
|
+
dependencies = [
|
|
18
|
+
"local-deep-research==1.6.11",
|
|
19
|
+
"cloakbrowser>=0.3.27",
|
|
20
|
+
"requests>=2.31",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.optional-dependencies]
|
|
24
|
+
mcp = ["mcp>=1.0"]
|
|
25
|
+
dev = ["pytest>=8.0", "ruff>=0.6"]
|
|
26
|
+
|
|
27
|
+
[project.scripts]
|
|
28
|
+
deepcloak = "deepcloak.cli:main"
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Homepage = "https://github.com/Mrbaeksang/deepcloak"
|
|
32
|
+
Repository = "https://github.com/Mrbaeksang/deepcloak"
|
|
33
|
+
Author = "https://baeksang.dev"
|
|
34
|
+
|
|
35
|
+
[tool.hatch.build.targets.wheel]
|
|
36
|
+
packages = ["src/deepcloak"]
|
|
37
|
+
|
|
38
|
+
# Keep the PyPI sdist lean — ship code + docs, not the demo media or showcase site.
|
|
39
|
+
[tool.hatch.build.targets.sdist]
|
|
40
|
+
include = ["src/deepcloak", "README.md", "LICENSE", "NOTICE", "CHANGELOG.md", "pyproject.toml"]
|
|
41
|
+
|
|
42
|
+
[tool.ruff]
|
|
43
|
+
line-length = 100
|
|
44
|
+
src = ["src", "tests"]
|
|
45
|
+
|
|
46
|
+
[tool.ruff.lint]
|
|
47
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
48
|
+
|
|
49
|
+
[tool.pytest.ini_options]
|
|
50
|
+
testpaths = ["tests"]
|
|
51
|
+
addopts = "-q"
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Sample data (real, from live runs)
|
|
2
|
+
|
|
3
|
+
Captured on `local-deep-research==1.6.11` + `cloakbrowser==0.3.31`, local Qwen3.6 + SearXNG.
|
|
4
|
+
|
|
5
|
+
- **`evidence.json`** — a real Bot Wall bypass demo: a Stealth Fetch **Bypassed** Cloudflare
|
|
6
|
+
Turnstile on `nowsecure.nl` (~5s), while `example.com` stayed on the fast plain path
|
|
7
|
+
(~0.1s). This is the escalation policy working: stealth only when a wall is detected.
|
|
8
|
+
- **`report.md`** — a sample cited report produced by a full research run
|
|
9
|
+
("How does Cloudflare Turnstile detect bots?") via the local LLM.
|
|
10
|
+
|
|
11
|
+
These are two separate demos (the report ran in `quick` depth, which is snippet-based and
|
|
12
|
+
doesn't fetch full pages — so its own evidence is empty; use `detailed`/`report` for
|
|
13
|
+
bypass-bearing runs). The static showcase site (issue #9) will render a single canonical
|
|
14
|
+
run that produces both together.
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Detect a Bot Wall in a plain HTTP response.
|
|
2
|
+
|
|
3
|
+
Pure module: ``classify()`` takes a status code, headers, and body and returns a
|
|
4
|
+
``BotWall`` (the kind + the signal that triggered it) or ``None``. A detected
|
|
5
|
+
Bot Wall is what triggers an Escalation in ``fetch_router``.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
__all__ = ["BotWall", "classify", "from_response"]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class BotWall:
|
|
19
|
+
kind: str # cloudflare | datadome | turnstile | recaptcha | rate_limit | blocked
|
|
20
|
+
signal: str # human-readable reason, recorded in the Evidence Record
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# Vendor markers checked in the body (lower-cased), strongest signal first.
|
|
24
|
+
_TURNSTILE = ("challenges.cloudflare.com/turnstile", "cf-turnstile")
|
|
25
|
+
_DATADOME = ("captcha-delivery.com", "protected by datadome", "geo.captcha-delivery")
|
|
26
|
+
_RECAPTCHA = ("g-recaptcha", "recaptcha/api.js", "google.com/recaptcha")
|
|
27
|
+
_CLOUDFLARE = (
|
|
28
|
+
"just a moment",
|
|
29
|
+
"checking your browser",
|
|
30
|
+
"/cdn-cgi/challenge-platform",
|
|
31
|
+
"attention required! | cloudflare",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def classify(status_code: int, headers: Mapping[str, str], body: str) -> BotWall | None:
|
|
36
|
+
body_l = (body or "").lower()
|
|
37
|
+
hdr_l = {str(k).lower(): str(v).lower() for k, v in dict(headers or {}).items()}
|
|
38
|
+
|
|
39
|
+
if any(m in body_l for m in _TURNSTILE):
|
|
40
|
+
return BotWall("turnstile", "Cloudflare Turnstile widget present")
|
|
41
|
+
if "datadome" in hdr_l.get("set-cookie", "") or "x-datadome" in hdr_l or any(
|
|
42
|
+
m in body_l for m in _DATADOME
|
|
43
|
+
):
|
|
44
|
+
return BotWall("datadome", "DataDome challenge present")
|
|
45
|
+
if any(m in body_l for m in _RECAPTCHA):
|
|
46
|
+
return BotWall("recaptcha", "reCAPTCHA widget present")
|
|
47
|
+
if any(m in body_l for m in _CLOUDFLARE):
|
|
48
|
+
return BotWall("cloudflare", "Cloudflare interstitial challenge")
|
|
49
|
+
if hdr_l.get("cf-mitigated") == "challenge":
|
|
50
|
+
return BotWall("cloudflare", "cf-mitigated: challenge header")
|
|
51
|
+
|
|
52
|
+
if status_code == 429:
|
|
53
|
+
return BotWall("rate_limit", "HTTP 429 Too Many Requests")
|
|
54
|
+
if status_code == 503 and ("cf-ray" in hdr_l or "cloudflare" in body_l):
|
|
55
|
+
return BotWall("cloudflare", "HTTP 503 from Cloudflare")
|
|
56
|
+
if status_code == 403:
|
|
57
|
+
return BotWall("blocked", "HTTP 403 Forbidden")
|
|
58
|
+
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def from_response(resp: Any) -> BotWall | None:
|
|
63
|
+
"""Convenience wrapper for a requests-style response object."""
|
|
64
|
+
return classify(
|
|
65
|
+
status_code=getattr(resp, "status_code", 0),
|
|
66
|
+
headers=getattr(resp, "headers", {}) or {},
|
|
67
|
+
body=getattr(resp, "text", "") or "",
|
|
68
|
+
)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""DeepCloak command-line interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
from . import __version__
|
|
9
|
+
|
|
10
|
+
__all__ = ["main", "build_parser"]
|
|
11
|
+
|
|
12
|
+
_SUBCOMMANDS = ("setup", "mcp")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
16
|
+
p = argparse.ArgumentParser(
|
|
17
|
+
prog="deepcloak",
|
|
18
|
+
description="Deep research that reads bot-walled pages.",
|
|
19
|
+
epilog="Subcommands: 'deepcloak setup' (install stealth browser), "
|
|
20
|
+
"'deepcloak mcp' (run MCP server).",
|
|
21
|
+
)
|
|
22
|
+
p.add_argument("--version", action="version", version=f"deepcloak {__version__}")
|
|
23
|
+
p.add_argument("query", nargs="?", help="The research question.")
|
|
24
|
+
p.add_argument("--depth", choices=["quick", "detailed", "report"], default="detailed")
|
|
25
|
+
p.add_argument("--engine", choices=["duckduckgo", "searxng", "auto"], default=None)
|
|
26
|
+
p.add_argument("--searxng-url", dest="searxng_url", default=None,
|
|
27
|
+
help="SearXNG instance URL (for --engine searxng)")
|
|
28
|
+
p.add_argument("--stealth", choices=["auto", "always", "off"], default=None)
|
|
29
|
+
p.add_argument("--respect-robots", action="store_true", dest="respect_robots")
|
|
30
|
+
p.add_argument("--proxy", default=None)
|
|
31
|
+
p.add_argument("--provider", default=None,
|
|
32
|
+
help="openai/anthropic/gemini/ollama/openai-endpoint")
|
|
33
|
+
p.add_argument("--model", default=None)
|
|
34
|
+
p.add_argument("--base-url", dest="base_url", default=None,
|
|
35
|
+
help="OpenAI-compatible URL (for openai-endpoint provider)")
|
|
36
|
+
p.add_argument("--out", default=None, help="Write the report to this file.")
|
|
37
|
+
return p
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _cli_dict(args: argparse.Namespace) -> dict:
|
|
41
|
+
keys = ("depth", "engine", "stealth", "respect_robots", "proxy",
|
|
42
|
+
"provider", "model", "base_url", "searxng_url", "out")
|
|
43
|
+
return {k: getattr(args, k) for k in keys if getattr(args, k) not in (None, False)}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def main(argv: list[str] | None = None) -> int:
|
|
47
|
+
argv = sys.argv[1:] if argv is None else list(argv)
|
|
48
|
+
|
|
49
|
+
# Subcommands are dispatched manually so the default `deepcloak "<query>"`
|
|
50
|
+
# form doesn't collide with an argparse subparser positional.
|
|
51
|
+
if argv and argv[0] == "setup":
|
|
52
|
+
from .setup import run_setup
|
|
53
|
+
|
|
54
|
+
return run_setup()
|
|
55
|
+
if argv and argv[0] == "mcp":
|
|
56
|
+
from .mcp_server import serve
|
|
57
|
+
|
|
58
|
+
serve()
|
|
59
|
+
return 0
|
|
60
|
+
|
|
61
|
+
args = build_parser().parse_args(argv)
|
|
62
|
+
if not args.query:
|
|
63
|
+
build_parser().print_help()
|
|
64
|
+
return 2
|
|
65
|
+
|
|
66
|
+
from .research_core import research
|
|
67
|
+
|
|
68
|
+
try:
|
|
69
|
+
result = research(args.query, cli=_cli_dict(args), verbose=True)
|
|
70
|
+
except Exception as exc: # surface a clean message, not a traceback
|
|
71
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
72
|
+
return 1
|
|
73
|
+
|
|
74
|
+
if args.out:
|
|
75
|
+
with open(args.out, "w", encoding="utf-8") as fh:
|
|
76
|
+
fh.write(result.report)
|
|
77
|
+
sidecar = f"{args.out}.evidence.json"
|
|
78
|
+
with open(sidecar, "w", encoding="utf-8") as fh:
|
|
79
|
+
fh.write(result.evidence_json)
|
|
80
|
+
print(f"wrote {args.out} (+ {sidecar})")
|
|
81
|
+
else:
|
|
82
|
+
print(result.report)
|
|
83
|
+
return 0
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
if __name__ == "__main__": # pragma: no cover
|
|
87
|
+
raise SystemExit(main())
|