crawlemoon 1.1.5__py3-none-any.whl

crawlemoon-1.1.5.dist-info/METADATA

@@ -0,0 +1,197 @@
+Metadata-Version: 2.4
+Name: crawlemoon
+Version: 1.1.5
+Summary: Advanced Web Crawling Platform with Deep Analysis and MCP Server
+Author-email: "emad.dev" <contact@emad.dev>
+License: MIT
+Keywords: crawling,scraping,automation,mcp,web-analysis,playwright,api-discovery
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Networking
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: playwright>=1.40.0
+Requires-Dist: mcp>=1.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pydantic-settings>=2.0.0
+Requires-Dist: aiohttp>=3.9.0
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: websockets>=12.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: python-json-logger>=2.0.0
+Requires-Dist: tenacity>=8.2.0
+Requires-Dist: cachetools>=5.3.0
+Requires-Dist: graphql-core>=3.2.0
+Requires-Dist: esprima>=4.0.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: lxml>=5.0.0
+Requires-Dist: cryptography>=41.0.0
+Requires-Dist: curl-cffi>=0.6.0
+Requires-Dist: fake-useragent>=1.4.0
+Requires-Dist: trafilatura>=1.6.0
+Requires-Dist: selectolax>=0.3.0
+Requires-Dist: markdownify>=0.11.0
+Requires-Dist: instructor>=1.0.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: python-Wappalyzer>=0.3.0
+Requires-Dist: ftfy>=6.1.0
+Requires-Dist: dateparser>=1.2.0
+Requires-Dist: url-normalize>=1.4.0
+Requires-Dist: tldextract>=5.0.0
+Provides-Extra: captcha
+Requires-Dist: python-anticaptcha>=1.0.0; extra == "captcha"
+Requires-Dist: capsolver>=1.0.0; extra == "captcha"
+Provides-Extra: ocr
+Requires-Dist: pytesseract>=0.3.10; extra == "ocr"
+Requires-Dist: Pillow>=10.0.0; extra == "ocr"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.7.0; extra == "dev"
+Requires-Dist: pip-audit>=2.7.0; extra == "dev"
+Requires-Dist: pre-commit>=3.6.0; extra == "dev"
+Requires-Dist: build>=1.0.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+Provides-Extra: all
+Requires-Dist: crawlemoon[captcha,dev,ocr]; extra == "all"
+Dynamic: license-file
+
+# Crawlemoon MCP Server
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/hero.png" alt="Crawlemoon MCP Server — free, AI-native web crawling for the agent era" width="100%"/>
+</p>
+
+<p align="left">
+  <img alt="python 3.10+ · pypi 1.1.0 · MIT · MCP-native · code style black" src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/badges.png" height="22"/>
+</p>
+
+A **free, open-source MCP server** that gives any agent (Claude Code, Cursor, Windsurf, …) **55 production-grade tools** for the full web-crawling stack: deep analysis, stealth, API discovery, session recording → runnable crawler, smart extraction. No proprietary API. No per-request fee.
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/features.png" alt="Crawlemoon capabilities — deep analysis, stealth, record→crawler, smart extraction" width="100%"/>
+</p>
+
+---
+
+## Quick start
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/install.png" alt="Three install paths — uvx, pipx, pip" width="100%"/>
+</p>
+
+The recommended path needs no install — `uvx` runs straight from PyPI:
+
+```json
+{
+  "mcpServers": {
+    "crawlemoon": {
+      "command": "uvx",
+      "args": ["crawlemoon-mcp-server"]
+    }
+  }
+}
+```
+
+> Requires [`uv`](https://docs.astral.sh/uv/getting-started/installation/). Install once: `curl -LsSf https://astral.sh/uv/install.sh | sh`. Or use `pipx run crawlemoon-mcp-server` / `pip install crawlemoon-mcp-server` instead.
+
+**Where to put that JSON:** Cursor → Settings → MCP. Claude Code → `~/.config/claude/mcp_settings.json`. Windsurf → Settings → MCP Servers.
+
+---
+
+## How it works
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/razavioo/crawlemoon-mcp-server/main/assets/architecture.png" alt="Agent → Crawlemoon → Browser/HTTP/Proxy → target web" width="100%"/>
+</p>
+
+Your agent talks to Crawlemoon over the Model Context Protocol. Crawlemoon owns a hardened browser pool, an HTTP stack with TLS fingerprinting, and a rotating proxy pool. While it fetches pages, it captures network traffic, reads scripts, and introspects schemas — so the agent gets clean structured data, not raw HTML.
+
+---
+
+## What's in the box
+
+A short list — see the source for the full set of 55 tools.
+
+| Group | Tools |
+|---|---|
+| **Deep analysis** | `deep_analyze`, `discover_apis`, `introspect_graphql`, `analyze_websocket`, `analyze_auth`, `detect_protection`, `detect_technology` |
+| **Stealth** | `stealth_request`, `configure_proxies`, `configure_rate_limit`, `add_proxy`, `test_proxy` |
+| **Record → crawler** | `record_session`, `stop_recording`, `export_recording`, `generate_crawler` |
+| **Extraction** | `smart_extract`, `extract_article`, `extract_tables`, `extract_links`, `extract_forms`, `extract_metadata`, `convert_to_markdown` |
+| **Page interaction** | `take_screenshot`, `fill_form`, `wait_and_extract`, `compare_pages`, `measure_performance`, `check_accessibility`, `get_dom_tree` |
+| **Sessions & cache** | `save_session`, `load_session`, `get_cookies`, `get_storage`, `clear_cache`, `get_cache_stats` |
+| **Advanced (opt-in)** | `execute_js`, `execute_cdp`, `deobfuscate_js`, `extract_from_js`, `solve_captcha` |
+
+---
+
+## Smart extraction — bring any LLM, including free ones
+
+`smart_extract` works **without any API key** using pattern matching. Plug in any OpenAI-compatible endpoint for higher accuracy — including FREE tiers:
+
+```bash
+# OpenRouter (free models exist)
+CRAWLEMOON_LLM_PROVIDER=openrouter
+CRAWLEMOON_LLM_API_KEY=sk-or-v1-xxx
+CRAWLEMOON_LLM_MODEL=meta-llama/llama-3.2-3b-instruct:free
+
+# Groq (free, very fast)
+CRAWLEMOON_LLM_PROVIDER=groq
+CRAWLEMOON_LLM_API_KEY=gsk_xxx
+
+# Local Ollama (no key needed)
+CRAWLEMOON_LLM_PROVIDER=ollama
+CRAWLEMOON_LLM_MODEL=llama3.2
+```
+
+Together, DeepSeek, Mistral, Fireworks, and standard OpenAI also work via `CRAWLEMOON_LLM_BASE_URL`.
+
+---
+
+## Configuration
+
+| Variable | Default | Notes |
+|---|---|---|
+| `CRAWLEMOON_HEADLESS` | `true` | Run browser without UI |
+| `CRAWLEMOON_BROWSER` | `chromium` | `chromium` / `firefox` / `webkit` |
+| `CRAWLEMOON_POOL_SIZE` | `5` | Max concurrent browsers |
+| `CRAWLEMOON_NAV_TIMEOUT` | `30.0` | Page-load timeout (s) |
+| `CRAWLEMOON_API_KEY` | _unset_ | If set, every tool call must include matching `_api_key` |
+| `CRAWLEMOON_ALLOW_DANGEROUS_JS` | `false` | Required for `execute_js` / `execute_cdp` / `deobfuscate_js` |
+| `CRAWLEMOON_JS_MAX_LENGTH` | `50000` | Length cap for JS payloads |
+| `CRAWLEMOON_JS_EXEC_TIMEOUT` | `10.0` | Per-script timeout (s) |
+
+---
+
+## Security
+
+`execute_js`, `execute_cdp`, and `deobfuscate_js` are **disabled by default** — they execute or operate on arbitrary code in a real browser. Enable on trusted networks with `CRAWLEMOON_ALLOW_DANGEROUS_JS=true`. Even then, payloads are length-capped, time-bounded, and a denylist rejects `eval`, `new Function`, dynamic `import()`, `document.write`, `importScripts`, and `WebAssembly.{compile,instantiate}`. Set `CRAWLEMOON_API_KEY` so MCP clients must present a matching `_api_key`.
+
+These are mitigations, not a sandbox: do not expose this server to untrusted clients.
+
+---
+
+## Develop
+
+```bash
+git clone https://github.com/razavioo/crawlemoon-mcp-server.git
+cd crawlemoon-mcp-server
+make dev-install      # editable install + dev/captcha/ocr extras + pre-commit
+make test             # pytest
+make lint             # ruff + mypy
+```
+
+PRs welcome. Particularly interested in: distributed mode (Redis queue), result sinks (Postgres / S3), Prometheus metrics. See [`MIT License`](LICENSE).
+
+<p align="center"><sub>Made by <a href="https://emad.dev">emad.dev</a></sub></p>

crawlemoon-1.1.5.dist-info/RECORD

@@ -0,0 +1,59 @@
+crawlemoon-1.1.5.dist-info/licenses/LICENSE,sha256=EoniT-RlPAS7H_GEDjPE8eYRQm1qnJZJEO6ZAbGQwC4,1067
+src/__init__.py,sha256=iIWAh-JSeKw1qmFotGpGCZNWNggK32prcQESGD8gmO0,1089
+src/exceptions.py,sha256=WZPkAf8zPqXBgE9TEhTxyym66HyKQgzzJtuywYyvatE,5946
+src/cli/__init__.py,sha256=JhLoLUvGogoGoTJEiA6EZO_3mbNEiciqgnUNUsNNNMI,44
+src/cli/main.py,sha256=8ZPOgBBS6CYKvlhFnssEAmVxbmo7SfURsj5vO4jBCis,11935
+src/core/__init__.py,sha256=DmypqdOcvJaawHp4_mSgLaGBujKPr7YkcEfBXVmn-TU,38
+src/core/logging.py,sha256=S-ZZmISmu_nb55zwQupI1av_Etwqd8JUDT0I0L0vhnk,4607
+src/core/rate_limiter.py,sha256=8ZgExSv0v0VnxOPU3by8ZJ36JY9PaYE9_Xf6IWg2o9A,13172
+src/core/recording_storage.py,sha256=h87cvXtkBcu0j394-EykYnFVTEs5tgSG-cPTPFPwT6o,14958
+src/core/browser/__init__.py,sha256=qZ3R3fK50GAZzHnmOn9Mzywih8u0LVXW5IVpuyH3FvA,49
+src/core/browser/cdp.py,sha256=OncjdlCeQOZOOggom5w02xBUlC1J1K2fzSBwU9dCqNU,3698
+src/core/browser/pool.py,sha256=rNsDDpcY1RbZ73mB4K_6t3Ei5mtmbAjLKiIVEDy-uIg,9978
+src/core/browser/proxy_pool.py,sha256=EAnue0GDyw4TpFQAS57pTHTtDMMIJhuUD4yneZfrSZo,11684
+src/core/browser/stealth.py,sha256=UifTLuLFzBw3v9eT06R_65aqkEiWlaGn6HS91Cuf56w,12574
+src/core/browser/xray.py,sha256=QXRdpPypYX713glAI_uwKucLEpqKEoo9-7AF0l-ppLM,32118
+src/core/cache/__init__.py,sha256=Mu6hiiovGgGteSKuGq0CSqKDchn4NA4kQSRGOvMo1TA,69
+src/core/cache/manager.py,sha256=zPwyc6b59XN-tfOpV04O-kN1mvXuh9HxwBf7tFZZ0Rc,12856
+src/core/http/__init__.py,sha256=LSnS10GFqnmrhQZsdQNSfMsUGqWhufE9wLgO6TIkz9Y,49
+src/core/http/stealth_client.py,sha256=Ng9QcaT8q_8vFlseQekwILEPwZHWyCrDlfwGPwskgh8,8220
+src/core/session/__init__.py,sha256=KlH3UErrJSDnr_VS-crzjM_EHU25W6FjaVgMTLO5ajM,49
+src/core/session/manager.py,sha256=otCV26_iCk-BUX6D5YyX-2bC03wLZ-AYA20FaTnL2ms,17859
+src/crawlers/__init__.py,sha256=rkv1aMsBjbCVOhDgKUK5uFgeZmPw12z2v8um1W_2XL0,34
+src/intelligence/__init__.py,sha256=H6W9IbRm3nsXmAfaYF_xX-6VsVvPaNm7P8QxSuFVvSo,54
+src/intelligence/extraction/__init__.py,sha256=avMYES_kTUGWxuVoJhMU4ZWz4bmQAxB7fuhttDofQWU,67
+src/intelligence/extraction/content.py,sha256=PXYN5TI5i5rJ4tn6O_Bk5NTy-JmTNFiebMOB2AoyszU,11400
+src/intelligence/extraction/smart.py,sha256=Gnb7kFghYFzv8YLJMyjXCQXjAAqdjcrP6s3BY_Y3WDA,18751
+src/intelligence/generator/__init__.py,sha256=jdGdyjr5QSU7fWq1sISn6bfmzqieNlapksMXq732ESg,69
+src/intelligence/generator/crawler_gen.py,sha256=6yg4V0L1H07tHj1WhPfI9WEtu438sU9TRsYDycaCldU,13935
+src/intelligence/js/__init__.py,sha256=-Us_025cYIR2-2cnq9CEbr6vb912W28D7ELsBn-TIoc,42
+src/intelligence/js/analyzer.py,sha256=TpCob6dDFnFmYwV-Hvmn1ok5Zv_5aGwb1saNGx5968I,11592
+src/intelligence/js/deobfuscator.py,sha256=mYir6TFKlqpdZjd9AUxCWSQ89cPzyG53iihIiITFaEo,6300
+src/intelligence/js/dynamic.py,sha256=XzqXBECn23x-saPoLPOQStdNZM1NhgxRn1-DtO_ENJk,3228
+src/intelligence/network/__init__.py,sha256=Ic9aTOwFv51xaGGH1fYjj3OBeY8Ncc66mYoJYLw50lQ,49
+src/intelligence/network/analyzer.py,sha256=NnseVEUubJJLWPGKiuz4ELKqvbVQmhg76qJJnxYTeFg,10002
+src/intelligence/network/api_discovery.py,sha256=DvbKlaRyslHzhLgr2S8TwzruJRk0WRHPHfeYMjZR_Js,19371
+src/intelligence/network/graphql.py,sha256=CFDq6l9mvVRYrPS3TE8bb7-haIKzyi3Jm18lThoYFto,1516
+src/intelligence/network/interceptor.py,sha256=hfzABtWBmOobhdpPUo5TL7fqeqap6YPn0YYdPZ685aQ,8326
+src/intelligence/network/sitemap.py,sha256=bwq-Gk_YKktVWwHNO0j4mt7ciauLsffugDn569yUKRU,8960
+src/intelligence/network/websocket.py,sha256=xpZcmnBHEqCmMd0nUQYmt2WnMkNDxkypjruBOj3eie4,1963
+src/intelligence/recorder/__init__.py,sha256=Hw4-OP8XDaRgucPimEzvzd7BeCqfsw2rXYaylurk0CU,44
+src/intelligence/recorder/session.py,sha256=Ubd1LsISbAvFFE6xquhhbx0Y1wWjgdO-dNGZCvo2T-Q,12443
+src/intelligence/recorder/state_machine.py,sha256=aLlBdj8L2ATStq_iI-25JWGrTIGGAbkKvh_URBbVS7g,3660
+src/intelligence/security/__init__.py,sha256=oR88l6-6Kva7sj6sT2O9sxQFs-uP2dni2cstidplMa4,46
+src/intelligence/security/auth.py,sha256=XeQAKpMR-Fql8ce1q2NRWD0UKa_CjTHwXvuobwUn-8Q,3046
+src/intelligence/security/bot_detection.py,sha256=Duk1KI43QiThIYohtCZkfgToIqIL2LEo5DuBPd7LOZ0,12493
+src/intelligence/security/captcha_solver.py,sha256=8wg0SKUQlFi-CjPXRRi5tML3HqA_G6_G8RsY32fNUmw,20211
+src/intelligence/security/technology_detector.py,sha256=noxyIErufHUNyvF7QaDh9jZXClVytR-uWnwZ0TFAItY,6651
+src/mcp/__init__.py,sha256=YswTJiEzV0hpeNe2GWxzpgDeFu52UZ86MTOra84UhyU,41
+src/mcp/config.py,sha256=znFr7x48lCoLSzbi3ouJms1Y-kMCLYQ1kamF5lNZ0SQ,6461
+src/mcp/schemas.py,sha256=z3jCyjw9B4xwie4K1f1kV4lKcvyE0tagK0SiUDflneQ,15315
+src/mcp/server.py,sha256=PwvzycgBbaMb-Uoq7ic59U7q4aafnooNp4UpJhSWO7o,180462
+src/mcp/utils.py,sha256=9qYja9u-TLqbDxMgcDoKYdATPami8ITIPrE_pbr6VIA,9833
+src/mcp/tools/__init__.py,sha256=PBZhxyYFpNQ_D99jtZAcqEahqdhS9Nhe5Mq12DCqFAo,33
+src/sites/__init__.py,sha256=gG9z7rEjQH1C4HQ36WLdLT4-FxxMD2-6Rj3yGxob7sI,45
+crawlemoon-1.1.5.dist-info/METADATA,sha256=qZdalWXG4GB5kaXLDycHi25EYOH97G5-m5lYqjNkoLE,8416
+crawlemoon-1.1.5.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+crawlemoon-1.1.5.dist-info/entry_points.txt,sha256=s3l8_-Bp73iim1643js2KlI5Tv3xZxmfaOZlBqauGZ4,131
+crawlemoon-1.1.5.dist-info/top_level.txt,sha256=74rtVfumQlgAPzR5_2CgYN24MB0XARCg0t-gzk6gTrM,4
+crawlemoon-1.1.5.dist-info/RECORD,,

crawlemoon-1.1.5.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

crawlemoon-1.1.5.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+crawl = src.cli.main:main
+crawlemoon = src.mcp.server:main_sync
+crawlemoon-mcp-server = src.mcp.server:main_sync

crawlemoon-1.1.5.dist-info/licenses/LICENSE

@@ -0,0 +1,23 @@
+MIT License
+
+Copyright (c) 2024 emad.dev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+

crawlemoon-1.1.5.dist-info/top_level.txt

@@ -0,0 +1 @@
+src

src/__init__.py

@@ -0,0 +1,47 @@
+"""Crawlemoon - Advanced Web Crawling Platform."""
+
+__version__ = "1.1.5"
+
+# Re-export the exception hierarchy so callers can use ``from src import CrawlemoonError``
+from .exceptions import (  # noqa: F401
+    CrawlemoonError,
+    BrowserError,
+    BrowserPoolError,
+    BrowserPoolExhaustedError,
+    BrowserInitError,
+    PageNavigationError,
+    PageInteractionError,
+    ProxyError,
+    NoHealthyProxyError,
+    ProxyTestError,
+    NetworkError,
+    HTTPRequestError,
+    RateLimitError,
+    ConnectionPoolError,
+    CacheError,
+    CacheBackendError,
+    CacheSerializationError,
+    SessionError,
+    SessionNotFoundError,
+    SessionEncryptionError,
+    SessionStorageError,
+    RecordingError,
+    RecordingNotFoundError,
+    RecordingSerializationError,
+    RecordingExpiredError,
+    AnalysisError,
+    APIDiscoveryError,
+    GraphQLError,
+    JSAnalysisError,
+    SitemapError,
+    BotDetectionError,
+    CaptchaError,
+    TechnologyDetectionError,
+    ContentExtractionError,
+    CrawlerGenerationError,
+    ConfigurationError,
+    ValidationError,
+    URLValidationError,
+)
+
+

src/cli/__init__.py

@@ -0,0 +1,9 @@
+"""CLI interface for Crawlemoon."""
+
+
+
+
+
+
+
+