wet-mcp 1.0.0__py3-none-any.whl

wet_mcp/__init__.py

@@ -0,0 +1,6 @@
+"""WET MCP Server - Web ExTract for AI Agents."""
+
+from wet_mcp.server import main, mcp
+
+__version__ = "0.1.0"
+__all__ = ["mcp", "main", "__version__"]

wet_mcp/__main__.py

@@ -0,0 +1,6 @@
+"""WET MCP Server entry point."""
+
+from wet_mcp.server import main
+
+if __name__ == "__main__":
+    main()

wet_mcp/config.py

@@ -0,0 +1,32 @@
+"""Configuration settings for WET MCP Server."""
+
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+    """WET MCP Server configuration."""
+
+    # SearXNG
+    searxng_url: str = "http://localhost:8080"
+    searxng_timeout: int = 30
+
+    # Crawler
+    crawler_headless: bool = True
+    crawler_timeout: int = 60
+
+    # Docker Management
+    wet_auto_docker: bool = True
+    wet_container_name: str = "wet-searxng"
+    wet_searxng_image: str = "searxng/searxng:latest"
+    wet_searxng_port: int = 8080
+
+    # Media
+    download_dir: str = "~/.wet-mcp/downloads"
+
+    # Logging
+    log_level: str = "INFO"
+
+    model_config = {"env_prefix": "", "case_sensitive": False}
+
+
+settings = Settings()

wet_mcp/docker_manager.py

@@ -0,0 +1,146 @@
+"""Docker container management for SearXNG."""
+
+import socket
+from importlib.resources import files
+from pathlib import Path
+
+from loguru import logger
+
+from wet_mcp.config import settings
+
+
+def _find_available_port(start_port: int, max_tries: int = 10) -> int:
+    """Find an available port starting from start_port."""
+    for offset in range(max_tries):
+        port = start_port + offset
+        try:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                s.bind(("localhost", port))
+                return port
+        except OSError:
+            continue
+    # Fallback to original port
+    return start_port
+
+
+def _get_settings_path() -> Path:
+    """Get path to SearXNG settings file.
+
+    Copies bundled settings.yml to user config directory for Docker mounting.
+    Uses ~/.wet-mcp/ which is typically shared with Docker.
+    """
+    config_dir = Path.home() / ".wet-mcp"
+    config_dir.mkdir(parents=True, exist_ok=True)
+
+    settings_file = config_dir / "searxng_settings.yml"
+
+    # Copy bundled settings if not exists
+    if not settings_file.exists():
+        bundled = files("wet_mcp").joinpath("searxng_settings.yml")
+        settings_file.write_text(bundled.read_text())
+        logger.debug(f"Copied SearXNG settings to: {settings_file}")
+
+    return settings_file
+
+
+def ensure_searxng() -> str:
+    """Start SearXNG container if not running. Returns URL.
+
+    This function handles:
+    - Automatic container creation if it doesn't exist
+    - Port conflict resolution (tries next available port)
+    - SearXNG configuration for JSON API format via settings.yml mount
+    - Graceful fallback to external SearXNG URL if Docker unavailable
+    """
+    if not settings.wet_auto_docker:
+        logger.info("Auto Docker disabled, using external SearXNG")
+        return settings.searxng_url
+
+    try:
+        from python_on_whales import DockerException, docker
+    except ImportError:
+        logger.warning("python-on-whales not installed, using external SearXNG")
+        return settings.searxng_url
+
+    container_name = settings.wet_container_name
+    image = settings.wet_searxng_image
+    preferred_port = settings.wet_searxng_port
+
+    try:
+        if docker.container.exists(container_name):
+            container = docker.container.inspect(container_name)
+            if container.state.running:
+                logger.debug(f"SearXNG container already running: {container_name}")
+                # Extract port from running container
+                ports = container.network_settings.ports
+                if ports and "8080/tcp" in ports and ports["8080/tcp"]:
+                    port = int(ports["8080/tcp"][0].get("HostPort", preferred_port))
+                else:
+                    port = preferred_port
+            else:
+                logger.info(f"Starting stopped container: {container_name}")
+                docker.container.start(container_name)
+                port = preferred_port
+        else:
+            # Find available port to avoid conflicts
+            port = _find_available_port(preferred_port)
+            if port != preferred_port:
+                logger.info(f"Port {preferred_port} in use, using {port}")
+
+            # Get settings file path
+            settings_path = _get_settings_path()
+
+            logger.info(f"Starting SearXNG container: {container_name}")
+            docker.run(
+                image,
+                name=container_name,
+                detach=True,
+                publish=[(port, 8080)],
+                volumes=[(str(settings_path), "/etc/searxng/settings.yml", "ro")],
+                envs={
+                    "SEARXNG_SECRET": "wet-internal",
+                },
+            )
+            logger.info(f"SearXNG container started on port {port}")
+
+        return f"http://localhost:{port}"
+
+    except DockerException as e:
+        logger.warning(f"Docker not available: {e}")
+        logger.warning("Falling back to external SearXNG URL")
+        return settings.searxng_url
+    except Exception as e:
+        logger.error(f"Failed to start SearXNG: {e}")
+        return settings.searxng_url
+
+
+def stop_searxng() -> None:
+    """Stop SearXNG container if running."""
+    if not settings.wet_auto_docker:
+        return
+
+    try:
+        from python_on_whales import docker
+
+        container_name = settings.wet_container_name
+        if docker.container.exists(container_name):
+            logger.info(f"Stopping container: {container_name}")
+            docker.container.stop(container_name)
+    except Exception as e:
+        logger.debug(f"Failed to stop container: {e}")
+
+
+def remove_searxng() -> None:
+    """Stop and remove SearXNG container."""
+    if not settings.wet_auto_docker:
+        return
+
+    try:
+        from python_on_whales import docker
+
+        container_name = settings.wet_container_name
+        if docker.container.exists(container_name):
+            logger.info(f"Removing container: {container_name}")
+            docker.container.remove(container_name, force=True)
+    except Exception as e:
+        logger.debug(f"Failed to remove container: {e}")

wet_mcp/docs/__init__.py

@@ -0,0 +1 @@
+"""Docs package for WET MCP Server."""

wet_mcp/docs/help.md

@@ -0,0 +1,55 @@
+# WET MCP Server - Help
+
+Welcome to **WET** (Web ExTract) MCP Server - an open-source alternative to Tavily.
+
+## Available Tools
+
+| Tool | Description |
+|:-----|:------------|
+| `web` | Web search, content extraction, crawling, site mapping |
+| `media` | Media discovery (images, videos, audio) and download |
+| `help` | Get full documentation for any tool |
+
+## Quick Reference
+
+### web tool
+
+```json
+// Search the web
+{"action": "search", "query": "your search query"}
+
+// Extract content from URLs
+{"action": "extract", "urls": ["https://example.com"]}
+
+// Crawl multiple pages
+{"action": "crawl", "urls": ["https://docs.example.com"], "depth": 2}
+
+// Map site structure
+{"action": "map", "urls": ["https://example.com"]}
+```
+
+### media tool
+
+```json
+// List media on a page
+{"action": "list", "url": "https://example.com"}
+
+// Download specific files
+{"action": "download", "media_urls": ["https://example.com/image.png"]}
+```
+
+## Getting Full Documentation
+
+Call `help` with the tool name:
+
+```json
+{"tool_name": "web"}    // Web tool documentation
+{"tool_name": "media"}  // Media tool documentation
+```
+
+## Features
+
+- **Auto-setup**: First run automatically installs Playwright and configures SearXNG
+- **Anti-bot bypass**: Stealth mode works with Cloudflare, Medium, LinkedIn, etc.
+- **Multimodal**: Extract and download images, videos, audio files
+- **Deep crawling**: Follow links to specified depth with page limits

wet_mcp/docs/media.md

@@ -0,0 +1,58 @@
+# media Tool Documentation
+
+Media discovery and download from web pages.
+
+## Actions
+
+### list
+Scan a page and return media URLs with metadata.
+
+**Parameters:**
+- `url` (required): Page URL to scan
+- `media_type`: Type of media - images, videos, audio, files, all (default: all)
+- `max_items`: Maximum items per type (default: 10)
+
+**Example:**
+```json
+{"action": "list", "url": "https://example.com/gallery", "media_type": "images"}
+```
+
+**Returns:**
+```json
+{
+  "images": [
+    {"src": "https://...", "alt": "...", "width": 800, "height": 600}
+  ],
+  "videos": [],
+  "audio": []
+}
+```
+
+---
+
+### download
+Download specific media files to local storage.
+
+**Parameters:**
+- `media_urls` (required): List of media URLs to download
+- `output_dir`: Output directory (default: ~/.wet-mcp/downloads)
+
+**Example:**
+```json
+{"action": "download", "media_urls": ["https://example.com/image.jpg"]}
+```
+
+**Returns:**
+```json
+[
+  {"url": "...", "path": "/path/to/file.jpg", "size": 12345}
+]
+```
+
+---
+
+## Workflow
+
+1. Use `list` to discover media on a page
+2. Review the results (optionally have AI analyze)
+3. Use `download` to save specific files locally

wet_mcp/docs/web.md

@@ -0,0 +1,73 @@
+# web Tool Documentation
+
+Web operations: search, extract, crawl, map.
+
+## Actions
+
+### search
+Web search via SearXNG metasearch engine.
+
+**Parameters:**
+- `query` (required): Search query string
+- `categories`: Search category - general, images, videos, files (default: general)
+- `max_results`: Maximum results to return (default: 10)
+
+**Example:**
+```json
+{"action": "search", "query": "python web scraping tutorial", "max_results": 5}
+```
+
+---
+
+### extract
+Get clean content from one or more URLs.
+
+**Parameters:**
+- `urls` (required): List of URLs to extract
+- `format`: Output format - markdown, text, html (default: markdown)
+- `stealth`: Enable stealth mode to bypass anti-bot (default: true)
+
+**Example:**
+```json
+{"action": "extract", "urls": ["https://example.com/article"]}
+```
+
+---
+
+### crawl
+Deep crawl starting from root URLs.
+
+**Parameters:**
+- `urls` (required): List of root URLs to crawl from
+- `depth`: How many levels deep to crawl (default: 2)
+- `max_pages`: Maximum pages to crawl (default: 20)
+- `format`: Output format (default: markdown)
+- `stealth`: Enable stealth mode (default: true)
+
+**Example:**
+```json
+{"action": "crawl", "urls": ["https://docs.example.com"], "depth": 3}
+```
+
+---
+
+### map
+Discover site structure without extracting content.
+
+**Parameters:**
+- `urls` (required): List of root URLs
+- `depth`: Discovery depth (default: 2)
+- `max_pages`: Maximum URLs to discover (default: 50)
+
+**Example:**
+```json
+{"action": "map", "urls": ["https://example.com"]}
+```
+
+---
+
+## Anti-Bot Features
+
+The `stealth` parameter enables:
+- Stealth mode: Masks navigator.webdriver, emulates plugins
+- Undetected browser: For advanced detection (Cloudflare, Datadome)

wet_mcp/searxng_settings.yml

@@ -0,0 +1,30 @@
+# SearXNG settings for WET MCP Server
+# Mounted into container at /etc/searxng/settings.yml
+
+use_default_settings: true
+
+server:
+  secret_key: "wet-mcp-internal-secret"
+  limiter: false
+  method: "GET"
+
+search:
+  formats:
+    - html
+    - json
+
+# Disable rate limiting for local use
+outgoing:
+  request_timeout: 10.0
+  max_request_timeout: 30.0
+
+# Engines to enable
+engines:
+  - name: google
+    disabled: false
+  - name: bing
+    disabled: false
+  - name: duckduckgo
+    disabled: false
+  - name: brave
+    disabled: false

wet_mcp/server.py

@@ -0,0 +1,172 @@
+"""WET MCP Server - Main server definition."""
+
+import sys
+from importlib.resources import files
+
+from loguru import logger
+from mcp.server.fastmcp import FastMCP
+
+from wet_mcp.config import settings
+from wet_mcp.docker_manager import ensure_searxng
+
+# Configure logging
+logger.remove()
+logger.add(sys.stderr, level=settings.log_level)
+
+# Initialize MCP server
+mcp = FastMCP(
+    name="wet",
+    instructions="Web ExTract MCP Server - search, extract, crawl, map with SearXNG",
+)
+
+# Store SearXNG URL after initialization
+_searxng_url: str | None = None
+
+
+def _get_searxng_url() -> str:
+    """Get SearXNG URL, initializing container if needed."""
+    global _searxng_url
+    if _searxng_url is None:
+        _searxng_url = ensure_searxng()
+    return _searxng_url
+
+
+@mcp.tool()
+async def web(
+    action: str,
+    query: str | None = None,
+    urls: list[str] | None = None,
+    categories: str = "general",
+    max_results: int = 10,
+    depth: int = 2,
+    max_pages: int = 20,
+    format: str = "markdown",
+    stealth: bool = True,
+) -> str:
+    """Web operations: search, extract, crawl, map.
+    - search: Web search via SearXNG (requires query)
+    - extract: Get clean content from URLs
+    - crawl: Deep crawl from root URL
+    - map: Discover site structure
+    Use `help` tool for full documentation.
+    """
+    from wet_mcp.sources.crawler import crawl, extract, sitemap
+    from wet_mcp.sources.searxng import search as searxng_search
+
+    match action:
+        case "search":
+            if not query:
+                return "Error: query is required for search action"
+            searxng_url = _get_searxng_url()
+            return await searxng_search(
+                searxng_url=searxng_url,
+                query=query,
+                categories=categories,
+                max_results=max_results,
+            )
+
+        case "extract":
+            if not urls:
+                return "Error: urls is required for extract action"
+            return await extract(
+                urls=urls,
+                format=format,
+                stealth=stealth,
+            )
+
+        case "crawl":
+            if not urls:
+                return "Error: urls is required for crawl action"
+            return await crawl(
+                urls=urls,
+                depth=depth,
+                max_pages=max_pages,
+                format=format,
+                stealth=stealth,
+            )
+
+        case "map":
+            if not urls:
+                return "Error: urls is required for map action"
+            return await sitemap(
+                urls=urls,
+                depth=depth,
+                max_pages=max_pages,
+            )
+
+        case _:
+            return f"Error: Unknown action '{action}'. Valid actions: search, extract, crawl, map"
+
+
+@mcp.tool()
+async def media(
+    action: str,
+    url: str | None = None,
+    media_type: str = "all",
+    media_urls: list[str] | None = None,
+    output_dir: str | None = None,
+    max_items: int = 10,
+) -> str:
+    """Media discovery and download.
+    - list: Scan page, return URLs + metadata
+    - download: Download specific files to local
+    MCP client decides whether to analyze media.
+    Use `help` tool for full documentation.
+    """
+    from wet_mcp.sources.crawler import download_media, list_media
+
+    match action:
+        case "list":
+            if not url:
+                return "Error: url is required for list action"
+            return await list_media(
+                url=url,
+                media_type=media_type,
+                max_items=max_items,
+            )
+
+        case "download":
+            if not media_urls:
+                return "Error: media_urls is required for download action"
+            return await download_media(
+                media_urls=media_urls,
+                output_dir=output_dir or settings.download_dir,
+            )
+
+        case _:
+            return f"Error: Unknown action '{action}'. Valid actions: list, download"
+
+
+@mcp.tool()
+async def help(tool_name: str = "web") -> str:
+    """Get full documentation for a tool.
+    Use when compressed descriptions are insufficient.
+    """
+    try:
+        doc_file = files("wet_mcp.docs").joinpath(f"{tool_name}.md")
+        return doc_file.read_text()
+    except FileNotFoundError:
+        return f"Error: No documentation found for tool '{tool_name}'"
+    except Exception as e:
+        return f"Error loading documentation: {e}"
+
+
+def main() -> None:
+    """Run the MCP server."""
+    from wet_mcp.setup import run_auto_setup
+
+    logger.info("Starting WET MCP Server...")
+
+    # Run auto-setup on first start (installs Playwright, etc.)
+    run_auto_setup()
+
+    # Initialize SearXNG container
+    searxng_url = _get_searxng_url()
+    logger.info(f"SearXNG URL: {searxng_url}")
+
+    # Run MCP server
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()

wet_mcp/setup.py

@@ -0,0 +1,94 @@
+"""Auto-setup utilities for WET MCP Server.
+
+This module handles automatic first-run setup:
+- Install Playwright browsers (chromium)
+- Verify Docker availability
+- Create configuration directories
+
+Setup runs automatically on first server start.
+"""
+
+import subprocess
+import sys
+from pathlib import Path
+
+from loguru import logger
+
+# Marker file to track if setup has been run
+SETUP_MARKER = Path.home() / ".wet-mcp" / ".setup-complete"
+
+
+def needs_setup() -> bool:
+    """Check if setup needs to run."""
+    return not SETUP_MARKER.exists()
+
+
+def run_auto_setup() -> bool:
+    """Run automatic setup on first start.
+
+    Returns:
+        True if setup succeeded or was already done, False on failure.
+    """
+    if not needs_setup():
+        logger.debug("Setup already complete, skipping")
+        return True
+
+    logger.info("First run detected, running auto-setup...")
+
+    success = True
+
+    # Step 1: Create config directory
+    config_dir = Path.home() / ".wet-mcp"
+    config_dir.mkdir(parents=True, exist_ok=True)
+    logger.debug(f"Created config directory: {config_dir}")
+
+    # Step 2: Install Playwright chromium (required for Crawl4AI)
+    logger.info("Installing Playwright chromium browser...")
+    try:
+        result = subprocess.run(
+            [sys.executable, "-m", "playwright", "install", "chromium"],
+            capture_output=True,
+            text=True,
+            timeout=300,
+        )
+        if result.returncode == 0:
+            logger.info("Playwright chromium installed successfully")
+        else:
+            logger.warning(f"Playwright install warning: {result.stderr[:200]}")
+            # Don't fail - might already be installed
+    except subprocess.TimeoutExpired:
+        logger.error("Playwright installation timed out")
+        success = False
+    except FileNotFoundError:
+        logger.warning("Playwright command not found, some features may not work")
+
+    # Step 3: Verify Docker (optional, for SearXNG)
+    try:
+        result = subprocess.run(
+            ["docker", "version", "--format", "{{.Server.Version}}"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        if result.returncode == 0:
+            logger.debug(f"Docker available: v{result.stdout.strip()}")
+        else:
+            logger.info("Docker not running, will use external SearXNG URL if configured")
+    except FileNotFoundError:
+        logger.info("Docker not installed, will use external SearXNG URL if configured")
+    except subprocess.TimeoutExpired:
+        logger.debug("Docker check timed out")
+
+    # Mark setup as complete
+    if success:
+        SETUP_MARKER.touch()
+        logger.info("Auto-setup complete!")
+
+    return success
+
+
+def reset_setup() -> None:
+    """Reset setup marker to force re-run on next start."""
+    if SETUP_MARKER.exists():
+        SETUP_MARKER.unlink()
+        logger.info("Setup marker removed, will re-run on next start")

wet_mcp/sources/__init__.py

@@ -0,0 +1 @@
+"""Sources package for WET MCP Server."""

wet_mcp/sources/crawler.py

@@ -0,0 +1,296 @@
+"""Crawl4AI integration for web crawling and extraction."""
+
+import json
+from pathlib import Path
+
+from loguru import logger
+
+
+async def extract(
+    urls: list[str],
+    format: str = "markdown",
+    stealth: bool = True,
+) -> str:
+    """Extract content from URLs.
+
+    Args:
+        urls: List of URLs to extract
+        format: Output format (markdown, text, html)
+        stealth: Enable stealth mode
+
+    Returns:
+        JSON string with extracted content
+    """
+    from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+    logger.info(f"Extracting content from {len(urls)} URLs")
+
+    browser_config = BrowserConfig(
+        headless=True,
+        enable_stealth=stealth,
+    )
+
+    results = []
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        for url in urls:
+            try:
+                result = await crawler.arun(
+                    url,
+                    config=CrawlerRunConfig(),
+                )
+
+                if result.success:
+                    content = result.markdown if format == "markdown" else result.cleaned_html
+                    results.append({
+                        "url": url,
+                        "title": result.metadata.get("title", ""),
+                        "content": content,
+                        "links": {
+                            "internal": result.links.get("internal", [])[:20],
+                            "external": result.links.get("external", [])[:20],
+                        },
+                    })
+                else:
+                    results.append({
+                        "url": url,
+                        "error": result.error_message or "Failed to extract",
+                    })
+
+            except Exception as e:
+                logger.error(f"Error extracting {url}: {e}")
+                results.append({
+                    "url": url,
+                    "error": str(e),
+                })
+
+    logger.info(f"Extracted {len(results)} pages")
+    return json.dumps(results, ensure_ascii=False, indent=2)
+
+
+async def crawl(
+    urls: list[str],
+    depth: int = 2,
+    max_pages: int = 20,
+    format: str = "markdown",
+    stealth: bool = True,
+) -> str:
+    """Deep crawl from root URLs.
+
+    Args:
+        urls: List of root URLs
+        depth: Crawl depth
+        max_pages: Maximum pages to crawl
+        format: Output format
+        stealth: Enable stealth mode
+
+    Returns:
+        JSON string with crawled content
+    """
+    from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+    logger.info(f"Crawling {len(urls)} URLs with depth={depth}")
+
+    browser_config = BrowserConfig(
+        headless=True,
+        enable_stealth=stealth,
+    )
+
+    all_results = []
+    visited = set()
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        for root_url in urls:
+            to_crawl = [(root_url, 0)]
+
+            while to_crawl and len(all_results) < max_pages:
+                url, current_depth = to_crawl.pop(0)
+
+                if url in visited or current_depth > depth:
+                    continue
+
+                visited.add(url)
+
+                try:
+                    result = await crawler.arun(
+                        url,
+                        config=CrawlerRunConfig(),
+                    )
+
+                    if result.success:
+                        content = result.markdown if format == "markdown" else result.cleaned_html
+                        all_results.append({
+                            "url": url,
+                            "depth": current_depth,
+                            "title": result.metadata.get("title", ""),
+                            "content": content[:5000],  # Limit content size
+                        })
+
+                        # Add internal links for next depth
+                        if current_depth < depth:
+                            internal_links = result.links.get("internal", [])
+                            for link_item in internal_links[:10]:
+                                # Crawl4AI returns dicts with 'href' key
+                                link_url = link_item.get("href", "") if isinstance(link_item, dict) else link_item
+                                if link_url and link_url not in visited:
+                                    to_crawl.append((link_url, current_depth + 1))
+
+                except Exception as e:
+                    logger.error(f"Error crawling {url}: {e}")
+
+    logger.info(f"Crawled {len(all_results)} pages")
+    return json.dumps(all_results, ensure_ascii=False, indent=2)
+
+
+async def sitemap(
+    urls: list[str],
+    depth: int = 2,
+    max_pages: int = 50,
+) -> str:
+    """Discover site structure.
+
+    Args:
+        urls: List of root URLs
+        depth: Discovery depth
+        max_pages: Maximum pages to discover
+
+    Returns:
+        JSON string with discovered URLs
+    """
+    from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+    logger.info(f"Mapping {len(urls)} URLs")
+
+    browser_config = BrowserConfig(headless=True)
+
+    all_urls = []
+    visited = set()
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        for root_url in urls:
+            to_visit = [(root_url, 0)]
+            site_urls = []
+
+            while to_visit and len(site_urls) < max_pages:
+                url, current_depth = to_visit.pop(0)
+
+                if url in visited or current_depth > depth:
+                    continue
+
+                visited.add(url)
+                site_urls.append({"url": url, "depth": current_depth})
+
+                try:
+                    result = await crawler.arun(
+                        url,
+                        config=CrawlerRunConfig(),
+                    )
+
+                    if result.success and current_depth < depth:
+                        for link in result.links.get("internal", [])[:20]:
+                            if link not in visited:
+                                to_visit.append((link, current_depth + 1))
+
+                except Exception as e:
+                    logger.debug(f"Error mapping {url}: {e}")
+
+            all_urls.extend(site_urls)
+
+    logger.info(f"Mapped {len(all_urls)} URLs")
+    return json.dumps(all_urls, ensure_ascii=False, indent=2)
+
+
+async def list_media(
+    url: str,
+    media_type: str = "all",
+    max_items: int = 10,
+) -> str:
+    """List media from a page.
+
+    Args:
+        url: Page URL to scan
+        media_type: Type of media (images, videos, audio, files, all)
+        max_items: Maximum items to return
+
+    Returns:
+        JSON string with media list
+    """
+    from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+    logger.info(f"Listing media from: {url}")
+
+    browser_config = BrowserConfig(headless=True)
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url,
+            config=CrawlerRunConfig(),
+        )
+
+        if not result.success:
+            return json.dumps({"error": result.error_message or "Failed to load page"})
+
+        media = result.media or {}
+
+        output = {}
+
+        if media_type in ("images", "all"):
+            output["images"] = media.get("images", [])[:max_items]
+        if media_type in ("videos", "all"):
+            output["videos"] = media.get("videos", [])[:max_items]
+        if media_type in ("audio", "all"):
+            # Crawl4AI uses 'audios' (plural)
+            output["audio"] = media.get("audios", [])[:max_items]
+
+        logger.info(f"Found media: {sum(len(v) for v in output.values())} items")
+        return json.dumps(output, ensure_ascii=False, indent=2)
+
+
+async def download_media(
+    media_urls: list[str],
+    output_dir: str,
+) -> str:
+    """Download media files.
+
+    Args:
+        media_urls: List of media URLs to download
+        output_dir: Output directory
+
+    Returns:
+        JSON string with download results
+    """
+    import httpx
+
+    logger.info(f"Downloading {len(media_urls)} media files")
+
+    output_path = Path(output_dir).expanduser()
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    results = []
+
+    async with httpx.AsyncClient(timeout=60) as client:
+        for url in media_urls:
+            try:
+                response = await client.get(url)
+                response.raise_for_status()
+
+                filename = url.split("/")[-1].split("?")[0] or "download"
+                filepath = output_path / filename
+
+                filepath.write_bytes(response.content)
+
+                results.append({
+                    "url": url,
+                    "path": str(filepath),
+                    "size": len(response.content),
+                })
+
+            except Exception as e:
+                logger.error(f"Error downloading {url}: {e}")
+                results.append({
+                    "url": url,
+                    "error": str(e),
+                })
+
+    logger.info(f"Downloaded {len([r for r in results if 'path' in r])} files")
+    return json.dumps(results, ensure_ascii=False, indent=2)

wet_mcp/sources/searxng.py

@@ -0,0 +1,72 @@
+"""SearXNG search integration."""
+
+import json
+
+import httpx
+from loguru import logger
+
+
+async def search(
+    searxng_url: str,
+    query: str,
+    categories: str = "general",
+    max_results: int = 10,
+) -> str:
+    """Search via SearXNG API.
+
+    Args:
+        searxng_url: SearXNG instance URL
+        query: Search query
+        categories: Search category (general, images, videos, files)
+        max_results: Maximum number of results
+
+    Returns:
+        JSON string with search results
+    """
+    logger.info(f"Searching SearXNG: {query}")
+
+    params = {
+        "q": query,
+        "format": "json",
+        "categories": categories,
+    }
+
+    try:
+        async with httpx.AsyncClient(timeout=30) as client:
+            response = await client.get(
+                f"{searxng_url}/search",
+                params=params,
+            )
+            response.raise_for_status()
+            data = response.json()
+
+        results = data.get("results", [])[:max_results]
+
+        # Format results
+        formatted = []
+        for r in results:
+            formatted.append({
+                "url": r.get("url", ""),
+                "title": r.get("title", ""),
+                "snippet": r.get("content", ""),
+                "source": r.get("engine", ""),
+            })
+
+        output = {
+            "results": formatted,
+            "total": len(formatted),
+            "query": query,
+        }
+
+        logger.info(f"Found {len(formatted)} results for: {query}")
+        return json.dumps(output, ensure_ascii=False, indent=2)
+
+    except httpx.HTTPStatusError as e:
+        logger.error(f"SearXNG HTTP error: {e}")
+        return json.dumps({"error": f"HTTP error: {e.response.status_code}"})
+    except httpx.RequestError as e:
+        logger.error(f"SearXNG request error: {e}")
+        return json.dumps({"error": f"Request error: {e}"})
+    except Exception as e:
+        logger.error(f"SearXNG error: {e}")
+        return json.dumps({"error": str(e)})

wet_mcp-1.0.0.dist-info/METADATA

@@ -0,0 +1,182 @@
+Metadata-Version: 2.4
+Name: wet-mcp
+Version: 1.0.0
+Summary: Open-source MCP Server thay thế Tavily - Web search, extract, crawl với SearXNG
+Project-URL: Homepage, https://github.com/n24q02m/wet-mcp
+Project-URL: Repository, https://github.com/n24q02m/wet-mcp.git
+Project-URL: Issues, https://github.com/n24q02m/wet-mcp/issues
+Author-email: n24q02m <quangminh2422004@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: crawl4ai,mcp,searxng,tavily-alternative,web-scraping
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: ==3.13.*
+Requires-Dist: crawl4ai>=0.8.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: loguru>=0.7.0
+Requires-Dist: mcp[cli]>=1.0.0
+Requires-Dist: pydantic-settings>=2.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: python-on-whales>=0.73.0
+Description-Content-Type: text/markdown
+
+# WET - Web ExTract MCP Server
+
+[![PyPI version](https://badge.fury.io/py/wet-mcp.svg)](https://badge.fury.io/py/wet-mcp)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+> **Open-source MCP Server thay thế Tavily cho web scraping & multimodal extraction**
+
+Zero-install experience: chỉ cần `uvx wet-mcp` - tự động setup và quản lý SearXNG container.
+
+## Features
+
+| Feature | Description |
+|:--------|:------------|
+| **Web Search** | Tìm kiếm qua SearXNG (metasearch: Google, Bing, DuckDuckGo, Brave) |
+| **Content Extract** | Trích xuất nội dung sạch (Markdown/Text/HTML) |
+| **Deep Crawl** | Đi qua nhiều trang con từ URL gốc với depth control |
+| **Site Map** | Khám phá cấu trúc URL của website |
+| **Media** | List và download images, videos, audio files |
+| **Anti-bot** | Stealth mode bypass Cloudflare, Medium, LinkedIn, Twitter |
+
+## Quick Start
+
+### Prerequisites
+
+- Docker daemon running (for SearXNG)
+- Python 3.13+ (hoặc dùng uvx)
+
+### MCP Client Configuration
+
+**Claude Desktop / Cursor / Windsurf / Antigravity:**
+
+```json
+{
+  "mcpServers": {
+    "wet": {
+      "command": "uvx",
+      "args": ["wet-mcp"]
+    }
+  }
+}
+```
+
+**Đó là tất cả!** Khi MCP client gọi wet-mcp lần đầu:
+1. Tự động install Playwright chromium
+2. Tự động pull SearXNG Docker image
+3. Start `wet-searxng` container
+4. Chạy MCP server
+
+### Without uvx
+
+```bash
+pip install wet-mcp
+wet-mcp
+```
+
+## Tools
+
+| Tool | Actions | Description |
+|:-----|:--------|:------------|
+| `web` | search, extract, crawl, map | Web operations |
+| `media` | list, download | Media discovery & download |
+| `help` | - | Full documentation |
+
+### Examples
+
+```python
+# Search
+{"action": "search", "query": "python web scraping", "max_results": 10}
+
+# Extract content
+{"action": "extract", "urls": ["https://example.com"]}
+
+# Crawl with depth
+{"action": "crawl", "urls": ["https://docs.python.org"], "depth": 2}
+
+# Map site structure
+{"action": "map", "urls": ["https://example.com"]}
+
+# List media
+{"action": "list", "url": "https://github.com/python/cpython"}
+
+# Download media
+{"action": "download", "media_urls": ["https://example.com/image.png"]}
+```
+
+## Tech Stack
+
+| Component | Technology |
+|:----------|:-----------|
+| Language | Python 3.13 |
+| MCP Framework | FastMCP |
+| Web Search | SearXNG (auto-managed Docker) |
+| Web Crawling | Crawl4AI |
+| Docker Management | python-on-whales |
+
+## How It Works
+
+```
+┌─────────────────────────────────────────────────────────┐
+│                    MCP Client                           │
+│            (Claude, Cursor, Windsurf)                   │
+└─────────────────────┬───────────────────────────────────┘
+                      │ MCP Protocol
+                      ▼
+┌─────────────────────────────────────────────────────────┐
+│                   WET MCP Server                        │
+│  ┌──────────┐  ┌──────────┐  ┌──────────────────────┐   │
+│  │   web    │  │  media   │  │        help          │   │
+│  │ (search, │  │ (list,   │  │  (full documentation)│   │
+│  │ extract, │  │ download)│  └──────────────────────┘   │
+│  │ crawl,   │  └────┬─────┘                             │
+│  │ map)     │       │                                   │
+│  └────┬─────┘       │                                   │
+│       │             │                                   │
+│       ▼             ▼                                   │
+│  ┌──────────┐  ┌──────────┐                             │
+│  │ SearXNG  │  │ Crawl4AI │                             │
+│  │ (Docker) │  │(Playwright)│                           │
+│  └──────────┘  └──────────┘                             │
+└─────────────────────────────────────────────────────────┘
+```
+
+## Configuration
+
+Environment variables:
+
+| Variable | Default | Description |
+|:---------|:--------|:------------|
+| `WET_AUTO_DOCKER` | `true` | Auto-manage SearXNG container |
+| `WET_SEARXNG_PORT` | `8080` | SearXNG container port |
+| `SEARXNG_URL` | `http://localhost:8080` | External SearXNG URL |
+| `LOG_LEVEL` | `INFO` | Logging level |
+
+## Container Management
+
+```bash
+# View SearXNG logs
+docker logs wet-searxng
+
+# Stop SearXNG
+docker stop wet-searxng
+
+# Remove container (will be recreated on next run)
+docker rm wet-searxng
+
+# Reset auto-setup (forces re-install Playwright)
+rm ~/.wet-mcp/.setup-complete
+```
+
+## License
+
+MIT License

wet_mcp-1.0.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+wet_mcp/__init__.py,sha256=I6YcMMy-wD5AY49rICs4lkd3tZQ4-XWkPpvoj7KwsKs,152
+wet_mcp/__main__.py,sha256=fjPYOwPnE1LDxkSbFZH0NjH8WLx8F2NjsO8vD-ujVbw,106
+wet_mcp/config.py,sha256=9ppAQSMmoNN5vAZHxSwMAjyeA7-3k3Nm6ANRUVMikBU,716
+wet_mcp/docker_manager.py,sha256=mFuzPO_9WjdTK8h7bqZ9VJ9NfW1Dkb7t5UAbTb_6GaE,5106
+wet_mcp/searxng_settings.yml,sha256=mB-AgqDGzoEG5xF5gHfPtH3s3TRyt3wV9FYC3b6wdIY,524
+wet_mcp/server.py,sha256=wmlZuAv6WvTrf-8LKYfNPvSI8vXk8Y0R5ZMuMPenNk4,4851
+wet_mcp/setup.py,sha256=8leo5QG_gRgVnpHhapOohbrsFoET4WaZ-cLEEc4Tvvw,2938
+wet_mcp/docs/__init__.py,sha256=JRSAzxxW76HaoAy6cf2lArX0abAhab3DiJtEwK4CYjc,39
+wet_mcp/docs/help.md,sha256=K91uveU6hM_tWBR9iqgn6_g0rt9h1h2zyEA3RMefeks,1428
+wet_mcp/docs/media.md,sha256=dktqg67x5yJaMXkHB2ShR9m3YaC2y56Ya82XKCVnl5k,1144
+wet_mcp/docs/web.md,sha256=Ts7hBo_TEBcfT10iscag9-PxAKUkhhiaywricUTj3to,1691
+wet_mcp/sources/__init__.py,sha256=NzIac1ha0nZR93Uivsq0GqBxdJrfm0q83IQAPeip4I4,42
+wet_mcp/sources/crawler.py,sha256=TBdH0raBoVVSohAHPv4J1DaaFbnjIkhLIf3SyI17tn8,9049
+wet_mcp/sources/searxng.py,sha256=Mg3WHy1z4OqvEJAOnn674S7ejCxBaROKBoBJJLulOxQ,1979
+wet_mcp-1.0.0.dist-info/METADATA,sha256=722R2KzbPcjtH_5zeeuM1nNaDDXy6Zo4OooZ6bjPC3c,6532
+wet_mcp-1.0.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+wet_mcp-1.0.0.dist-info/entry_points.txt,sha256=MvjtmQDh--zOPfnE-21Q861RFRLkE1xDbcTGAgURT_Y,41
+wet_mcp-1.0.0.dist-info/licenses/LICENSE,sha256=d7xQ6sRyeGus6gnvwgqiQtSY7XdFw0Jd0w5-Co_xHnk,1064
+wet_mcp-1.0.0.dist-info/RECORD,,

wet_mcp-1.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

wet_mcp-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+wet-mcp = wet_mcp:main

wet_mcp-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 n24q02m
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.