crawlemoon 1.1.5__tar.gz

crawlemoon-1.1.5/LICENSE

@@ -0,0 +1,23 @@
+MIT License
+
+Copyright (c) 2024 emad.dev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+

crawlemoon-1.1.5/PKG-INFO

@@ -0,0 +1,197 @@
+Metadata-Version: 2.4
+Name: crawlemoon
+Version: 1.1.5
+Summary: Advanced Web Crawling Platform with Deep Analysis and MCP Server
+Author-email: "emad.dev" <contact@emad.dev>
+License: MIT
+Keywords: crawling,scraping,automation,mcp,web-analysis,playwright,api-discovery
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Networking
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: playwright>=1.40.0
+Requires-Dist: mcp>=1.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pydantic-settings>=2.0.0
+Requires-Dist: aiohttp>=3.9.0
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: websockets>=12.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: python-json-logger>=2.0.0
+Requires-Dist: tenacity>=8.2.0
+Requires-Dist: cachetools>=5.3.0
+Requires-Dist: graphql-core>=3.2.0
+Requires-Dist: esprima>=4.0.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: lxml>=5.0.0
+Requires-Dist: cryptography>=41.0.0
+Requires-Dist: curl-cffi>=0.6.0
+Requires-Dist: fake-useragent>=1.4.0
+Requires-Dist: trafilatura>=1.6.0
+Requires-Dist: selectolax>=0.3.0
+Requires-Dist: markdownify>=0.11.0
+Requires-Dist: instructor>=1.0.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: python-Wappalyzer>=0.3.0
+Requires-Dist: ftfy>=6.1.0
+Requires-Dist: dateparser>=1.2.0
+Requires-Dist: url-normalize>=1.4.0
+Requires-Dist: tldextract>=5.0.0
+Provides-Extra: captcha
+Requires-Dist: python-anticaptcha>=1.0.0; extra == "captcha"
+Requires-Dist: capsolver>=1.0.0; extra == "captcha"
+Provides-Extra: ocr
+Requires-Dist: pytesseract>=0.3.10; extra == "ocr"
+Requires-Dist: Pillow>=10.0.0; extra == "ocr"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.7.0; extra == "dev"
+Requires-Dist: pip-audit>=2.7.0; extra == "dev"
+Requires-Dist: pre-commit>=3.6.0; extra == "dev"
+Requires-Dist: build>=1.0.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+Provides-Extra: all
+Requires-Dist: crawlemoon[captcha,dev,ocr]; extra == "all"
+Dynamic: license-file
+
+# Crawlemoon MCP Server
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/hero.png" alt="Crawlemoon MCP Server — free, AI-native web crawling for the agent era" width="100%"/>
+</p>
+
+<p align="left">
+  <img alt="python 3.10+ · pypi 1.1.0 · MIT · MCP-native · code style black" src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/badges.png" height="22"/>
+</p>
+
+A **free, open-source MCP server** that gives any agent (Claude Code, Cursor, Windsurf, …) **55 production-grade tools** for the full web-crawling stack: deep analysis, stealth, API discovery, session recording → runnable crawler, smart extraction. No proprietary API. No per-request fee.
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/features.png" alt="Crawlemoon capabilities — deep analysis, stealth, record→crawler, smart extraction" width="100%"/>
+</p>
+
+---
+
+## Quick start
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/install.png" alt="Three install paths — uvx, pipx, pip" width="100%"/>
+</p>
+
+The recommended path needs no install — `uvx` runs straight from PyPI:
+
+```json
+{
+  "mcpServers": {
+    "crawlemoon": {
+      "command": "uvx",
+      "args": ["crawlemoon-mcp-server"]
+    }
+  }
+}
+```
+
+> Requires [`uv`](https://docs.astral.sh/uv/getting-started/installation/). Install once: `curl -LsSf https://astral.sh/uv/install.sh | sh`. Or use `pipx run crawlemoon-mcp-server` / `pip install crawlemoon-mcp-server` instead.
+
+**Where to put that JSON:** Cursor → Settings → MCP. Claude Code → `~/.config/claude/mcp_settings.json`. Windsurf → Settings → MCP Servers.
+
+---
+
+## How it works
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/architecture.png" alt="Agent → Crawlemoon → Browser/HTTP/Proxy → target web" width="100%"/>
+</p>
+
+Your agent talks to Crawlemoon over the Model Context Protocol. Crawlemoon owns a hardened browser pool, an HTTP stack with TLS fingerprinting, and a rotating proxy pool. While it fetches pages, it captures network traffic, reads scripts, and introspects schemas — so the agent gets clean structured data, not raw HTML.
+
+---
+
+## What's in the box
+
+A short list — see the source for the full set of 55 tools.
+
+| Group | Tools |
+|---|---|
+| **Deep analysis** | `deep_analyze`, `discover_apis`, `introspect_graphql`, `analyze_websocket`, `analyze_auth`, `detect_protection`, `detect_technology` |
+| **Stealth** | `stealth_request`, `configure_proxies`, `configure_rate_limit`, `add_proxy`, `test_proxy` |
+| **Record → crawler** | `record_session`, `stop_recording`, `export_recording`, `generate_crawler` |
+| **Extraction** | `smart_extract`, `extract_article`, `extract_tables`, `extract_links`, `extract_forms`, `extract_metadata`, `convert_to_markdown` |
+| **Page interaction** | `take_screenshot`, `fill_form`, `wait_and_extract`, `compare_pages`, `measure_performance`, `check_accessibility`, `get_dom_tree` |
+| **Sessions & cache** | `save_session`, `load_session`, `get_cookies`, `get_storage`, `clear_cache`, `get_cache_stats` |
+| **Advanced (opt-in)** | `execute_js`, `execute_cdp`, `deobfuscate_js`, `extract_from_js`, `solve_captcha` |
+
+---
+
+## Smart extraction — bring any LLM, including free ones
+
+`smart_extract` works **without any API key** using pattern matching. Plug in any OpenAI-compatible endpoint for higher accuracy — including FREE tiers:
+
+```bash
+# OpenRouter (free models exist)
+CRAWLEMOON_LLM_PROVIDER=openrouter
+CRAWLEMOON_LLM_API_KEY=sk-or-v1-xxx
+CRAWLEMOON_LLM_MODEL=meta-llama/llama-3.2-3b-instruct:free
+
+# Groq (free, very fast)
+CRAWLEMOON_LLM_PROVIDER=groq
+CRAWLEMOON_LLM_API_KEY=gsk_xxx
+
+# Local Ollama (no key needed)
+CRAWLEMOON_LLM_PROVIDER=ollama
+CRAWLEMOON_LLM_MODEL=llama3.2
+```
+
+Together, DeepSeek, Mistral, Fireworks, and standard OpenAI also work via `CRAWLEMOON_LLM_BASE_URL`.
+
+---
+
+## Configuration
+
+| Variable | Default | Notes |
+|---|---|---|
+| `CRAWLEMOON_HEADLESS` | `true` | Run browser without UI |
+| `CRAWLEMOON_BROWSER` | `chromium` | `chromium` / `firefox` / `webkit` |
+| `CRAWLEMOON_POOL_SIZE` | `5` | Max concurrent browsers |
+| `CRAWLEMOON_NAV_TIMEOUT` | `30.0` | Page-load timeout (s) |
+| `CRAWLEMOON_API_KEY` | _unset_ | If set, every tool call must include matching `_api_key` |
+| `CRAWLEMOON_ALLOW_DANGEROUS_JS` | `false` | Required for `execute_js` / `execute_cdp` / `deobfuscate_js` |
+| `CRAWLEMOON_JS_MAX_LENGTH` | `50000` | Length cap for JS payloads |
+| `CRAWLEMOON_JS_EXEC_TIMEOUT` | `10.0` | Per-script timeout (s) |
+
+---
+
+## Security
+
+`execute_js`, `execute_cdp`, and `deobfuscate_js` are **disabled by default** — they execute or operate on arbitrary code in a real browser. Enable on trusted networks with `CRAWLEMOON_ALLOW_DANGEROUS_JS=true`. Even then, payloads are length-capped, time-bounded, and a denylist rejects `eval`, `new Function`, dynamic `import()`, `document.write`, `importScripts`, and `WebAssembly.{compile,instantiate}`. Set `CRAWLEMOON_API_KEY` so MCP clients must present a matching `_api_key`.
+
+These are mitigations, not a sandbox: do not expose this server to untrusted clients.
+
+---
+
+## Develop
+
+```bash
+git clone https://github.com/razavioo/crawlemoon-mcp-server.git
+cd crawlemoon-mcp-server
+make dev-install      # editable install + dev/captcha/ocr extras + pre-commit
+make test             # pytest
+make lint             # ruff + mypy
+```
+
+PRs welcome. Particularly interested in: distributed mode (Redis queue), result sinks (Postgres / S3), Prometheus metrics. See [`MIT License`](LICENSE).
+
+<p align="center"><sub>Made by <a href="https://emad.dev">emad.dev</a></sub></p>

crawlemoon-1.1.5/README.md

@@ -0,0 +1,128 @@
+# Crawlemoon MCP Server
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/hero.png" alt="Crawlemoon MCP Server — free, AI-native web crawling for the agent era" width="100%"/>
+</p>
+
+<p align="left">
+  <img alt="python 3.10+ · pypi 1.1.0 · MIT · MCP-native · code style black" src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/badges.png" height="22"/>
+</p>
+
+A **free, open-source MCP server** that gives any agent (Claude Code, Cursor, Windsurf, …) **55 production-grade tools** for the full web-crawling stack: deep analysis, stealth, API discovery, session recording → runnable crawler, smart extraction. No proprietary API. No per-request fee.
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/features.png" alt="Crawlemoon capabilities — deep analysis, stealth, record→crawler, smart extraction" width="100%"/>
+</p>
+
+---
+
+## Quick start
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/install.png" alt="Three install paths — uvx, pipx, pip" width="100%"/>
+</p>
+
+The recommended path needs no install — `uvx` runs straight from PyPI:
+
+```json
+{
+  "mcpServers": {
+    "crawlemoon": {
+      "command": "uvx",
+      "args": ["crawlemoon-mcp-server"]
+    }
+  }
+}
+```
+
+> Requires [`uv`](https://docs.astral.sh/uv/getting-started/installation/). Install once: `curl -LsSf https://astral.sh/uv/install.sh | sh`. Or use `pipx run crawlemoon-mcp-server` / `pip install crawlemoon-mcp-server` instead.
+
+**Where to put that JSON:** Cursor → Settings → MCP. Claude Code → `~/.config/claude/mcp_settings.json`. Windsurf → Settings → MCP Servers.
+
+---
+
+## How it works
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/architecture.png" alt="Agent → Crawlemoon → Browser/HTTP/Proxy → target web" width="100%"/>
+</p>
+
+Your agent talks to Crawlemoon over the Model Context Protocol. Crawlemoon owns a hardened browser pool, an HTTP stack with TLS fingerprinting, and a rotating proxy pool. While it fetches pages, it captures network traffic, reads scripts, and introspects schemas — so the agent gets clean structured data, not raw HTML.
+
+---
+
+## What's in the box
+
+A short list — see the source for the full set of 55 tools.
+
+| Group | Tools |
+|---|---|
+| **Deep analysis** | `deep_analyze`, `discover_apis`, `introspect_graphql`, `analyze_websocket`, `analyze_auth`, `detect_protection`, `detect_technology` |
+| **Stealth** | `stealth_request`, `configure_proxies`, `configure_rate_limit`, `add_proxy`, `test_proxy` |
+| **Record → crawler** | `record_session`, `stop_recording`, `export_recording`, `generate_crawler` |
+| **Extraction** | `smart_extract`, `extract_article`, `extract_tables`, `extract_links`, `extract_forms`, `extract_metadata`, `convert_to_markdown` |
+| **Page interaction** | `take_screenshot`, `fill_form`, `wait_and_extract`, `compare_pages`, `measure_performance`, `check_accessibility`, `get_dom_tree` |
+| **Sessions & cache** | `save_session`, `load_session`, `get_cookies`, `get_storage`, `clear_cache`, `get_cache_stats` |
+| **Advanced (opt-in)** | `execute_js`, `execute_cdp`, `deobfuscate_js`, `extract_from_js`, `solve_captcha` |
+
+---
+
+## Smart extraction — bring any LLM, including free ones
+
+`smart_extract` works **without any API key** using pattern matching. Plug in any OpenAI-compatible endpoint for higher accuracy — including FREE tiers:
+
+```bash
+# OpenRouter (free models exist)
+CRAWLEMOON_LLM_PROVIDER=openrouter
+CRAWLEMOON_LLM_API_KEY=sk-or-v1-xxx
+CRAWLEMOON_LLM_MODEL=meta-llama/llama-3.2-3b-instruct:free
+
+# Groq (free, very fast)
+CRAWLEMOON_LLM_PROVIDER=groq
+CRAWLEMOON_LLM_API_KEY=gsk_xxx
+
+# Local Ollama (no key needed)
+CRAWLEMOON_LLM_PROVIDER=ollama
+CRAWLEMOON_LLM_MODEL=llama3.2
+```
+
+Together, DeepSeek, Mistral, Fireworks, and standard OpenAI also work via `CRAWLEMOON_LLM_BASE_URL`.
+
+---
+
+## Configuration
+
+| Variable | Default | Notes |
+|---|---|---|
+| `CRAWLEMOON_HEADLESS` | `true` | Run browser without UI |
+| `CRAWLEMOON_BROWSER` | `chromium` | `chromium` / `firefox` / `webkit` |
+| `CRAWLEMOON_POOL_SIZE` | `5` | Max concurrent browsers |
+| `CRAWLEMOON_NAV_TIMEOUT` | `30.0` | Page-load timeout (s) |
+| `CRAWLEMOON_API_KEY` | _unset_ | If set, every tool call must include matching `_api_key` |
+| `CRAWLEMOON_ALLOW_DANGEROUS_JS` | `false` | Required for `execute_js` / `execute_cdp` / `deobfuscate_js` |
+| `CRAWLEMOON_JS_MAX_LENGTH` | `50000` | Length cap for JS payloads |
+| `CRAWLEMOON_JS_EXEC_TIMEOUT` | `10.0` | Per-script timeout (s) |
+
+---
+
+## Security
+
+`execute_js`, `execute_cdp`, and `deobfuscate_js` are **disabled by default** — they execute or operate on arbitrary code in a real browser. Enable on trusted networks with `CRAWLEMOON_ALLOW_DANGEROUS_JS=true`. Even then, payloads are length-capped, time-bounded, and a denylist rejects `eval`, `new Function`, dynamic `import()`, `document.write`, `importScripts`, and `WebAssembly.{compile,instantiate}`. Set `CRAWLEMOON_API_KEY` so MCP clients must present a matching `_api_key`.
+
+These are mitigations, not a sandbox: do not expose this server to untrusted clients.
+
+---
+
+## Develop
+
+```bash
+git clone https://github.com/razavioo/crawlemoon-mcp-server.git
+cd crawlemoon-mcp-server
+make dev-install      # editable install + dev/captcha/ocr extras + pre-commit
+make test             # pytest
+make lint             # ruff + mypy
+```
+
+PRs welcome. Particularly interested in: distributed mode (Redis queue), result sinks (Postgres / S3), Prometheus metrics. See [`MIT License`](LICENSE).
+
+<p align="center"><sub>Made by <a href="https://emad.dev">emad.dev</a></sub></p>

crawlemoon-1.1.5/crawlemoon.egg-info/PKG-INFO

@@ -0,0 +1,197 @@
+Metadata-Version: 2.4
+Name: crawlemoon
+Version: 1.1.5
+Summary: Advanced Web Crawling Platform with Deep Analysis and MCP Server
+Author-email: "emad.dev" <contact@emad.dev>
+License: MIT
+Keywords: crawling,scraping,automation,mcp,web-analysis,playwright,api-discovery
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Networking
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: playwright>=1.40.0
+Requires-Dist: mcp>=1.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pydantic-settings>=2.0.0
+Requires-Dist: aiohttp>=3.9.0
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: websockets>=12.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: python-json-logger>=2.0.0
+Requires-Dist: tenacity>=8.2.0
+Requires-Dist: cachetools>=5.3.0
+Requires-Dist: graphql-core>=3.2.0
+Requires-Dist: esprima>=4.0.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: lxml>=5.0.0
+Requires-Dist: cryptography>=41.0.0
+Requires-Dist: curl-cffi>=0.6.0
+Requires-Dist: fake-useragent>=1.4.0
+Requires-Dist: trafilatura>=1.6.0
+Requires-Dist: selectolax>=0.3.0
+Requires-Dist: markdownify>=0.11.0
+Requires-Dist: instructor>=1.0.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: python-Wappalyzer>=0.3.0
+Requires-Dist: ftfy>=6.1.0
+Requires-Dist: dateparser>=1.2.0
+Requires-Dist: url-normalize>=1.4.0
+Requires-Dist: tldextract>=5.0.0
+Provides-Extra: captcha
+Requires-Dist: python-anticaptcha>=1.0.0; extra == "captcha"
+Requires-Dist: capsolver>=1.0.0; extra == "captcha"
+Provides-Extra: ocr
+Requires-Dist: pytesseract>=0.3.10; extra == "ocr"
+Requires-Dist: Pillow>=10.0.0; extra == "ocr"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.7.0; extra == "dev"
+Requires-Dist: pip-audit>=2.7.0; extra == "dev"
+Requires-Dist: pre-commit>=3.6.0; extra == "dev"
+Requires-Dist: build>=1.0.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+Provides-Extra: all
+Requires-Dist: crawlemoon[captcha,dev,ocr]; extra == "all"
+Dynamic: license-file
+
+# Crawlemoon MCP Server
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/hero.png" alt="Crawlemoon MCP Server — free, AI-native web crawling for the agent era" width="100%"/>
+</p>
+
+<p align="left">
+  <img alt="python 3.10+ · pypi 1.1.0 · MIT · MCP-native · code style black" src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/badges.png" height="22"/>
+</p>
+
+A **free, open-source MCP server** that gives any agent (Claude Code, Cursor, Windsurf, …) **55 production-grade tools** for the full web-crawling stack: deep analysis, stealth, API discovery, session recording → runnable crawler, smart extraction. No proprietary API. No per-request fee.
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/features.png" alt="Crawlemoon capabilities — deep analysis, stealth, record→crawler, smart extraction" width="100%"/>
+</p>
+
+---
+
+## Quick start
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/install.png" alt="Three install paths — uvx, pipx, pip" width="100%"/>
+</p>
+
+The recommended path needs no install — `uvx` runs straight from PyPI:
+
+```json
+{
+  "mcpServers": {
+    "crawlemoon": {
+      "command": "uvx",
+      "args": ["crawlemoon-mcp-server"]
+    }
+  }
+}
+```
+
+> Requires [`uv`](https://docs.astral.sh/uv/getting-started/installation/). Install once: `curl -LsSf https://astral.sh/uv/install.sh | sh`. Or use `pipx run crawlemoon-mcp-server` / `pip install crawlemoon-mcp-server` instead.
+
+**Where to put that JSON:** Cursor → Settings → MCP. Claude Code → `~/.config/claude/mcp_settings.json`. Windsurf → Settings → MCP Servers.
+
+---
+
+## How it works
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/architecture.png" alt="Agent → Crawlemoon → Browser/HTTP/Proxy → target web" width="100%"/>
+</p>
+
+Your agent talks to Crawlemoon over the Model Context Protocol. Crawlemoon owns a hardened browser pool, an HTTP stack with TLS fingerprinting, and a rotating proxy pool. While it fetches pages, it captures network traffic, reads scripts, and introspects schemas — so the agent gets clean structured data, not raw HTML.
+
+---
+
+## What's in the box
+
+A short list — see the source for the full set of 55 tools.
+
+| Group | Tools |
+|---|---|
+| **Deep analysis** | `deep_analyze`, `discover_apis`, `introspect_graphql`, `analyze_websocket`, `analyze_auth`, `detect_protection`, `detect_technology` |
+| **Stealth** | `stealth_request`, `configure_proxies`, `configure_rate_limit`, `add_proxy`, `test_proxy` |
+| **Record → crawler** | `record_session`, `stop_recording`, `export_recording`, `generate_crawler` |
+| **Extraction** | `smart_extract`, `extract_article`, `extract_tables`, `extract_links`, `extract_forms`, `extract_metadata`, `convert_to_markdown` |
+| **Page interaction** | `take_screenshot`, `fill_form`, `wait_and_extract`, `compare_pages`, `measure_performance`, `check_accessibility`, `get_dom_tree` |
+| **Sessions & cache** | `save_session`, `load_session`, `get_cookies`, `get_storage`, `clear_cache`, `get_cache_stats` |
+| **Advanced (opt-in)** | `execute_js`, `execute_cdp`, `deobfuscate_js`, `extract_from_js`, `solve_captcha` |
+
+---
+
+## Smart extraction — bring any LLM, including free ones
+
+`smart_extract` works **without any API key** using pattern matching. Plug in any OpenAI-compatible endpoint for higher accuracy — including FREE tiers:
+
+```bash
+# OpenRouter (free models exist)
+CRAWLEMOON_LLM_PROVIDER=openrouter
+CRAWLEMOON_LLM_API_KEY=sk-or-v1-xxx
+CRAWLEMOON_LLM_MODEL=meta-llama/llama-3.2-3b-instruct:free
+
+# Groq (free, very fast)
+CRAWLEMOON_LLM_PROVIDER=groq
+CRAWLEMOON_LLM_API_KEY=gsk_xxx
+
+# Local Ollama (no key needed)
+CRAWLEMOON_LLM_PROVIDER=ollama
+CRAWLEMOON_LLM_MODEL=llama3.2
+```
+
+Together, DeepSeek, Mistral, Fireworks, and standard OpenAI also work via `CRAWLEMOON_LLM_BASE_URL`.
+
+---
+
+## Configuration
+
+| Variable | Default | Notes |
+|---|---|---|
+| `CRAWLEMOON_HEADLESS` | `true` | Run browser without UI |
+| `CRAWLEMOON_BROWSER` | `chromium` | `chromium` / `firefox` / `webkit` |
+| `CRAWLEMOON_POOL_SIZE` | `5` | Max concurrent browsers |
+| `CRAWLEMOON_NAV_TIMEOUT` | `30.0` | Page-load timeout (s) |
+| `CRAWLEMOON_API_KEY` | _unset_ | If set, every tool call must include matching `_api_key` |
+| `CRAWLEMOON_ALLOW_DANGEROUS_JS` | `false` | Required for `execute_js` / `execute_cdp` / `deobfuscate_js` |
+| `CRAWLEMOON_JS_MAX_LENGTH` | `50000` | Length cap for JS payloads |
+| `CRAWLEMOON_JS_EXEC_TIMEOUT` | `10.0` | Per-script timeout (s) |
+
+---
+
+## Security
+
+`execute_js`, `execute_cdp`, and `deobfuscate_js` are **disabled by default** — they execute or operate on arbitrary code in a real browser. Enable on trusted networks with `CRAWLEMOON_ALLOW_DANGEROUS_JS=true`. Even then, payloads are length-capped, time-bounded, and a denylist rejects `eval`, `new Function`, dynamic `import()`, `document.write`, `importScripts`, and `WebAssembly.{compile,instantiate}`. Set `CRAWLEMOON_API_KEY` so MCP clients must present a matching `_api_key`.
+
+These are mitigations, not a sandbox: do not expose this server to untrusted clients.
+
+---
+
+## Develop
+
+```bash
+git clone https://github.com/razavioo/crawlemoon-mcp-server.git
+cd crawlemoon-mcp-server
+make dev-install      # editable install + dev/captcha/ocr extras + pre-commit
+make test             # pytest
+make lint             # ruff + mypy
+```
+
+PRs welcome. Particularly interested in: distributed mode (Redis queue), result sinks (Postgres / S3), Prometheus metrics. See [`MIT License`](LICENSE).
+
+<p align="center"><sub>Made by <a href="https://emad.dev">emad.dev</a></sub></p>

crawlemoon-1.1.5/crawlemoon.egg-info/SOURCES.txt

@@ -0,0 +1,94 @@
+LICENSE
+README.md
+pyproject.toml
+setup.py
+crawlemoon.egg-info/PKG-INFO
+crawlemoon.egg-info/SOURCES.txt
+crawlemoon.egg-info/dependency_links.txt
+crawlemoon.egg-info/entry_points.txt
+crawlemoon.egg-info/requires.txt
+crawlemoon.egg-info/top_level.txt
+src/__init__.py
+src/exceptions.py
+src/cli/__init__.py
+src/cli/main.py
+src/core/__init__.py
+src/core/logging.py
+src/core/rate_limiter.py
+src/core/recording_storage.py
+src/core/browser/__init__.py
+src/core/browser/cdp.py
+src/core/browser/pool.py
+src/core/browser/proxy_pool.py
+src/core/browser/stealth.py
+src/core/browser/xray.py
+src/core/cache/__init__.py
+src/core/cache/manager.py
+src/core/http/__init__.py
+src/core/http/stealth_client.py
+src/core/session/__init__.py
+src/core/session/manager.py
+src/crawlers/__init__.py
+src/intelligence/__init__.py
+src/intelligence/extraction/__init__.py
+src/intelligence/extraction/content.py
+src/intelligence/extraction/smart.py
+src/intelligence/generator/__init__.py
+src/intelligence/generator/crawler_gen.py
+src/intelligence/js/__init__.py
+src/intelligence/js/analyzer.py
+src/intelligence/js/deobfuscator.py
+src/intelligence/js/dynamic.py
+src/intelligence/network/__init__.py
+src/intelligence/network/analyzer.py
+src/intelligence/network/api_discovery.py
+src/intelligence/network/graphql.py
+src/intelligence/network/interceptor.py
+src/intelligence/network/sitemap.py
+src/intelligence/network/websocket.py
+src/intelligence/recorder/__init__.py
+src/intelligence/recorder/session.py
+src/intelligence/recorder/state_machine.py
+src/intelligence/security/__init__.py
+src/intelligence/security/auth.py
+src/intelligence/security/bot_detection.py
+src/intelligence/security/captcha_solver.py
+src/intelligence/security/technology_detector.py
+src/mcp/__init__.py
+src/mcp/config.py
+src/mcp/schemas.py
+src/mcp/server.py
+src/mcp/utils.py
+src/mcp/tools/__init__.py
+src/sites/__init__.py
+tests/test_api_discovery.py
+tests/test_auth_analyzer.py
+tests/test_bot_detection.py
+tests/test_browser_pool.py
+tests/test_cache_manager.py
+tests/test_captcha_solver.py
+tests/test_cdp_client.py
+tests/test_content_extractor.py
+tests/test_crawl_and_extract.py
+tests/test_crawler_generator.py
+tests/test_improvements.py
+tests/test_js_analyzer.py
+tests/test_js_deobfuscator.py
+tests/test_js_security.py
+tests/test_mcp_config.py
+tests/test_mcp_server.py
+tests/test_mcp_utils.py
+tests/test_network_interceptor.py
+tests/test_proxy_pool.py
+tests/test_rate_limiter.py
+tests/test_recording_storage.py
+tests/test_request_analyzer.py
+tests/test_session_manager.py
+tests/test_session_recorder.py
+tests/test_sitemap_analyzer.py
+tests/test_smart_extractor.py
+tests/test_stealth.py
+tests/test_stealth_client.py
+tests/test_technology_detector.py
+tests/test_xray.py
+tests/test_xray_advanced.py

crawlemoon-1.1.5/crawlemoon.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

crawlemoon-1.1.5/crawlemoon.egg-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+crawl = src.cli.main:main
+crawlemoon = src.mcp.server:main_sync
+crawlemoon-mcp-server = src.mcp.server:main_sync