groundhog-mcp 0.1.0__tar.gz

groundhog_mcp-0.1.0/.gitignore

@@ -0,0 +1,34 @@
+# Dependencies
+node_modules/
+**/node_modules/
+.venv/
+__pycache__/
+*.py[cod]
+
+# Build / runtime
+chrome-user-data/
+downloads/
+screenshots/
+*.log
+
+# Example/test run artifacts
+examples/**/*.png
+package-lock.json
+go.sum
+
+# Env / secrets
+.env
+.env.*
+*.pem
+*.key
+
+# OS / editor
+.DS_Store
+.idea/
+.vscode/
+*.swp
+
+# Internal (never publish)
+.superpowers/
+docs/superpowers/
+.internal/

groundhog_mcp-0.1.0/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: groundhog-mcp
+Version: 0.1.0
+Summary: Stealth-browser grounding tools for AI agents, over MCP
+Project-URL: Homepage, https://github.com/dmytrome/groundhog
+Project-URL: Repository, https://github.com/dmytrome/groundhog
+Project-URL: Issues, https://github.com/dmytrome/groundhog/issues
+Author: dmytrome
+License-Expression: MIT
+Keywords: agents,browser,cdp,mcp,scraping,ssrf,stealth
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
+Requires-Python: >=3.11
+Requires-Dist: mcp<2,>=1.28
+Requires-Dist: playwright>=1.49
+Requires-Dist: tldextract>=5.1
+Requires-Dist: trafilatura>=2.0

groundhog_mcp-0.1.0/pyproject.toml

@@ -0,0 +1,48 @@
+[project]
+name = "groundhog-mcp"
+version = "0.1.0"
+description = "Stealth-browser grounding tools for AI agents, over MCP"
+requires-python = ">=3.11"
+license = "MIT"
+authors = [{ name = "dmytrome" }]
+keywords = ["mcp", "stealth", "browser", "cdp", "scraping", "ssrf", "agents"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Topic :: Internet :: WWW/HTTP :: Browsers",
+]
+dependencies = [
+    "mcp>=1.28,<2",
+    "playwright>=1.49",
+    "trafilatura>=2.0",
+    "tldextract>=5.1",
+]
+
+[project.urls]
+Homepage = "https://github.com/dmytrome/groundhog"
+Repository = "https://github.com/dmytrome/groundhog"
+Issues = "https://github.com/dmytrome/groundhog/issues"
+
+[project.scripts]
+groundhog-mcp = "groundhog_mcp.__main__:main"
+
+[dependency-groups]
+dev = ["pytest>=8", "pytest-asyncio>=0.24", "ruff>=0.14"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP"]
+
+[tool.ruff.lint.isort]
+combine-as-imports = true

groundhog_mcp-0.1.0/src/groundhog_mcp/__main__.py

@@ -0,0 +1,11 @@
+from .server import build_server
+
+
+def main() -> None:
+    # stdio is the only transport for now; HTTP support ships with its own
+    # host/DNS-rebinding hardening.
+    build_server().run(transport="stdio")
+
+
+if __name__ == "__main__":
+    main()

groundhog_mcp-0.1.0/src/groundhog_mcp/config.py

@@ -0,0 +1,36 @@
+import os
+from dataclasses import dataclass
+
+DEFAULT_USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36"
+)
+
+
+@dataclass(frozen=True)
+class Config:
+    cdp_url: str
+    min_delay_ms: int
+    block_private_ips: bool
+    max_tokens: int
+    user_agent: str
+    auto_start_browser: bool
+    compose_file: str | None
+
+
+def _bool(value: str | None, default: bool) -> bool:
+    if value is None:
+        return default
+    return value.strip().lower() in ("1", "true", "yes", "on")
+
+
+def load_config() -> Config:
+    return Config(
+        cdp_url=os.environ.get("CDP_URL", "http://127.0.0.1:9222"),
+        min_delay_ms=int(os.environ.get("GROUNDHOG_MIN_DELAY_MS", "5000")),
+        block_private_ips=_bool(os.environ.get("GROUNDHOG_BLOCK_PRIVATE_IPS"), True),
+        max_tokens=int(os.environ.get("GROUNDHOG_MAX_TOKENS", "20000")),
+        user_agent=os.environ.get("GROUNDHOG_USER_AGENT", DEFAULT_USER_AGENT),
+        auto_start_browser=_bool(os.environ.get("GROUNDHOG_AUTO_START_BROWSER"), False),
+        compose_file=os.environ.get("GROUNDHOG_COMPOSE_FILE") or None,
+    )

groundhog_mcp-0.1.0/src/groundhog_mcp/engine.py

@@ -0,0 +1,156 @@
+import asyncio
+import urllib.request
+from dataclasses import dataclass
+
+import tldextract
+from playwright.async_api import (
+    Browser,
+    BrowserContext,
+    Error as PlaywrightError,
+    Playwright,
+    async_playwright,
+)
+
+from . import safety
+from .config import Config, load_config
+from .ratelimit import RateLimiter
+
+_GOTO_TIMEOUT_MS = 60_000
+_PROBE_TIMEOUT_S = 2.0
+_AUTOSTART_READY_TRIES = 30
+_ERR_DETAIL_CHARS = 300
+
+
+class BrowserUnavailableError(Exception):
+    """The stealth browser's CDP endpoint could not be reached."""
+
+
+def remediation(cfg: Config) -> str:
+    return (
+        f"Cannot reach the stealth browser at {cfg.cdp_url}. "
+        "Start it with `docker compose up -d` (see the Groundhog README), "
+        "or set GROUNDHOG_AUTO_START_BROWSER=true to let Groundhog start it for you."
+    )
+
+
+async def check_browser(cdp_url: str, timeout: float = _PROBE_TIMEOUT_S) -> bool:
+    """Return whether the CDP endpoint answers, via its `/json/version` probe."""
+    probe_url = cdp_url.rstrip("/") + "/json/version"
+
+    def _probe() -> bool:
+        try:
+            with urllib.request.urlopen(probe_url, timeout=timeout) as resp:
+                return resp.status == 200
+        except OSError:
+            return False  # refused / DNS / timeout all mean "not reachable"
+
+    return await asyncio.get_running_loop().run_in_executor(None, _probe)
+
+
+async def _start_browser_container(cfg: Config) -> None:
+    cmd = ["docker", "compose"]
+    if cfg.compose_file:
+        cmd += ["-f", cfg.compose_file]
+    cmd += ["up", "-d"]
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+        )
+    except FileNotFoundError as exc:
+        raise BrowserUnavailableError(
+            "GROUNDHOG_AUTO_START_BROWSER is set but Docker was not found on PATH. "
+            + remediation(cfg)
+        ) from exc
+    _, stderr = await proc.communicate()
+    if proc.returncode != 0:
+        detail = stderr.decode(errors="replace").strip()[:_ERR_DETAIL_CHARS]
+        raise BrowserUnavailableError(f"`docker compose up -d` failed: {detail}")
+    for _ in range(_AUTOSTART_READY_TRIES):
+        if await check_browser(cfg.cdp_url):
+            return
+        await asyncio.sleep(1)
+    raise BrowserUnavailableError(
+        f"Browser container started but {cfg.cdp_url} did not become ready in time."
+    )
+
+
+@dataclass
+class RenderedPage:
+    html: str
+    text: str
+    final_url: str
+    title: str
+
+
+def registrable_domain(url: str) -> str:
+    ext = tldextract.extract(url)
+    return ext.registered_domain or ext.fqdn or url
+
+
+class EngineProvider:
+    def __init__(self, cfg: Config):
+        self._cfg = cfg
+        self._pw: Playwright | None = None
+        self._browser: Browser | None = None
+        self._context: BrowserContext | None = None
+        self._rl = RateLimiter(cfg.min_delay_ms / 1000)
+
+    async def start(self) -> None:
+        self._pw = await async_playwright().start()
+        self._browser = await self._connect()
+        self._context = await self._browser.new_context(
+            user_agent=self._cfg.user_agent,
+            viewport={"width": 1920, "height": 1080},
+        )
+
+    async def _connect(self) -> Browser:
+        try:
+            return await self._pw.chromium.connect_over_cdp(self._cfg.cdp_url)
+        except PlaywrightError as exc:
+            if not self._cfg.auto_start_browser:
+                raise BrowserUnavailableError(remediation(self._cfg)) from exc
+        await _start_browser_container(self._cfg)
+        try:
+            return await self._pw.chromium.connect_over_cdp(self._cfg.cdp_url)
+        except PlaywrightError as exc:
+            raise BrowserUnavailableError(remediation(self._cfg)) from exc
+
+    async def fetch(self, url: str) -> RenderedPage:
+        await safety.check_url(url, self._cfg)
+        await self._rl.acquire(registrable_domain(url))
+        page = await self._context.new_page()
+        try:
+            await page.goto(url, wait_until="domcontentloaded", timeout=_GOTO_TIMEOUT_MS)
+            # A page can redirect to an internal address the initial check never
+            # saw; re-check the final URL so its content is never returned.
+            await safety.check_url(page.url, self._cfg)
+            return RenderedPage(
+                html=await page.content(),
+                text=await page.inner_text("body"),
+                final_url=page.url,
+                title=await page.title(),
+            )
+        finally:
+            await page.close()
+
+    async def aclose(self) -> None:
+        if self._context is not None:
+            await self._context.close()
+        if self._browser is not None:
+            await self._browser.close()
+        if self._pw is not None:
+            await self._pw.stop()
+
+
+_provider: EngineProvider | None = None
+_provider_lock = asyncio.Lock()
+
+
+async def get_provider() -> EngineProvider:
+    global _provider
+    async with _provider_lock:
+        if _provider is None:
+            provider = EngineProvider(load_config())
+            await provider.start()
+            _provider = provider
+    return _provider

groundhog_mcp-0.1.0/src/groundhog_mcp/extract.py

@@ -0,0 +1,27 @@
+import trafilatura
+
+_CHARS_PER_TOKEN = 4
+_TRUNCATION_MARKER = "\n\n[... truncated]"
+
+
+def truncate(text: str, max_tokens: int) -> tuple[str, bool]:
+    limit = max_tokens * _CHARS_PER_TOKEN
+    if len(text) <= limit:
+        return text, False
+    cut = text.rfind("\n\n", 0, limit)
+    if cut <= 0:
+        cut = limit
+    return text[:cut].rstrip() + _TRUNCATION_MARKER, True
+
+
+def to_markdown(html: str, text_fallback: str, url: str, max_tokens: int) -> tuple[str, bool]:
+    markdown = trafilatura.extract(
+        html,
+        url=url,
+        output_format="markdown",
+        include_links=True,
+        include_tables=True,
+    )
+    if not markdown:
+        markdown = text_fallback or ""
+    return truncate(markdown, max_tokens)

groundhog_mcp-0.1.0/src/groundhog_mcp/ratelimit.py

@@ -0,0 +1,28 @@
+import asyncio
+import time
+
+
+class RateLimiter:
+    """Enforces a minimum delay between acquisitions sharing the same key."""
+
+    def __init__(self, min_delay: float, *, clock=time.monotonic, sleep=asyncio.sleep):
+        self._min_delay = min_delay
+        self._clock = clock
+        self._sleep = sleep
+        self._last: dict[str, float] = {}
+        self._locks: dict[str, asyncio.Lock] = {}
+
+    def _lock(self, key: str) -> asyncio.Lock:
+        lock = self._locks.get(key)
+        if lock is None:
+            lock = self._locks[key] = asyncio.Lock()
+        return lock
+
+    async def acquire(self, key: str) -> None:
+        async with self._lock(key):
+            last = self._last.get(key)
+            if last is not None:
+                wait = self._min_delay - (self._clock() - last)
+                if wait > 0:
+                    await self._sleep(wait)
+            self._last[key] = self._clock()

groundhog_mcp-0.1.0/src/groundhog_mcp/safety.py

@@ -0,0 +1,40 @@
+import asyncio
+import ipaddress
+from urllib.parse import urlparse
+
+from .config import Config
+
+_CGNAT = ipaddress.ip_network("100.64.0.0/10")
+
+
+class BlockedURLError(Exception):
+    """Raised when a URL is disallowed by the SSRF guard."""
+
+
+def is_blocked_ip(ip: str) -> bool:
+    addr = ipaddress.ip_address(ip)
+    if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
+        addr = addr.ipv4_mapped
+    if (addr.is_private or addr.is_loopback or addr.is_link_local
+            or addr.is_reserved or addr.is_multicast or addr.is_unspecified):
+        return True
+    return isinstance(addr, ipaddress.IPv4Address) and addr in _CGNAT
+
+
+async def check_url(url: str, cfg: Config) -> None:
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        raise BlockedURLError(f"scheme not allowed: {parsed.scheme!r}")
+    if parsed.username or parsed.password:
+        raise BlockedURLError("credentials in URL are not allowed")
+    host = parsed.hostname
+    if not host:
+        raise BlockedURLError("URL has no host")
+    if not cfg.block_private_ips:
+        return
+    loop = asyncio.get_running_loop()
+    infos = await loop.getaddrinfo(host, None)
+    for info in infos:
+        ip = info[4][0]
+        if is_blocked_ip(ip):
+            raise BlockedURLError(f"blocked address: {host} -> {ip}")

groundhog_mcp-0.1.0/src/groundhog_mcp/server.py

@@ -0,0 +1,11 @@
+from mcp.server.fastmcp import FastMCP
+
+from .tools.read_url import read_url
+from .tools.status import status
+
+
+def build_server() -> FastMCP:
+    mcp = FastMCP("groundhog")
+    mcp.tool()(read_url)
+    mcp.tool()(status)
+    return mcp

groundhog_mcp-0.1.0/src/groundhog_mcp/tools/read_url.py

@@ -0,0 +1,42 @@
+from datetime import UTC, datetime
+from typing import TypedDict
+
+from .. import extract
+from ..config import load_config
+from ..engine import get_provider
+
+_FORMATS = ("markdown", "text")
+
+
+class ReadResult(TypedDict):
+    markdown: str
+    title: str
+    url: str
+    final_url: str
+    fetched_at: str
+    truncated: bool
+
+
+async def read_url(url: str, format: str = "markdown", max_tokens: int | None = None) -> ReadResult:
+    """Fetch a web page through the stealth browser and return clean Markdown
+    plus provenance (source URL, final URL, title, fetch time). Use this to
+    ground answers in live web content, including sites that block plain
+    fetchers. `format` may be "markdown" (default) or "text"."""
+    if format not in _FORMATS:
+        raise ValueError(f"format must be one of {_FORMATS}, got {format!r}")
+    cfg = load_config()
+    provider = await get_provider()
+    page = await provider.fetch(url)
+    limit = max_tokens or cfg.max_tokens
+    if format == "text":
+        body, truncated = extract.truncate(page.text, limit)
+    else:
+        body, truncated = extract.to_markdown(page.html, page.text, page.final_url, limit)
+    return {
+        "markdown": body,
+        "title": page.title,
+        "url": url,
+        "final_url": page.final_url,
+        "fetched_at": datetime.now(UTC).isoformat(),
+        "truncated": truncated,
+    }

groundhog_mcp-0.1.0/src/groundhog_mcp/tools/status.py

@@ -0,0 +1,20 @@
+from typing import TypedDict
+
+from ..config import load_config
+from ..engine import check_browser, remediation
+
+
+class StatusResult(TypedDict):
+    browser_reachable: bool
+    cdp_url: str
+    hint: str | None
+
+
+async def status() -> StatusResult:
+    """Check whether Groundhog can reach the stealth browser. Call this to
+    diagnose setup before fetching: if `browser_reachable` is false, follow
+    `hint` to start the browser, then retry."""
+    cfg = load_config()
+    reachable = await check_browser(cfg.cdp_url)
+    hint = None if reachable else remediation(cfg)
+    return {"browser_reachable": reachable, "cdp_url": cfg.cdp_url, "hint": hint}

groundhog_mcp-0.1.0/tests/test_config.py

@@ -0,0 +1,23 @@
+from groundhog_mcp.config import load_config
+
+
+def test_defaults(monkeypatch):
+    for key in ("CDP_URL", "GROUNDHOG_MIN_DELAY_MS", "GROUNDHOG_BLOCK_PRIVATE_IPS",
+                "GROUNDHOG_MAX_TOKENS", "GROUNDHOG_USER_AGENT"):
+        monkeypatch.delenv(key, raising=False)
+    cfg = load_config()
+    assert cfg.cdp_url == "http://127.0.0.1:9222"
+    assert cfg.min_delay_ms == 5000
+    assert cfg.block_private_ips is True
+    assert cfg.max_tokens == 20000
+    assert "Chrome/149" in cfg.user_agent
+
+
+def test_env_overrides(monkeypatch):
+    monkeypatch.setenv("CDP_URL", "http://127.0.0.1:9333")
+    monkeypatch.setenv("GROUNDHOG_MIN_DELAY_MS", "0")
+    monkeypatch.setenv("GROUNDHOG_BLOCK_PRIVATE_IPS", "false")
+    cfg = load_config()
+    assert cfg.cdp_url == "http://127.0.0.1:9333"
+    assert cfg.min_delay_ms == 0
+    assert cfg.block_private_ips is False

groundhog_mcp-0.1.0/tests/test_engine_live.py

@@ -0,0 +1,35 @@
+import os
+
+import pytest
+
+from groundhog_mcp.config import load_config
+from groundhog_mcp.engine import EngineProvider
+from groundhog_mcp.safety import BlockedURLError
+
+pytestmark = pytest.mark.skipif(
+    os.environ.get("RUN_LIVE") != "1",
+    reason="requires the engine running; set RUN_LIVE=1 and CDP_URL",
+)
+
+
+async def test_fetch_example_com():
+    provider = EngineProvider(load_config())
+    await provider.start()
+    try:
+        page = await provider.fetch("https://example.com/")
+        assert "Example Domain" in page.title
+        assert "Example Domain" in page.html
+        assert page.final_url.startswith("https://example.com")
+        assert "Example Domain" in page.text
+    finally:
+        await provider.aclose()
+
+
+async def test_fetch_blocks_internal():
+    provider = EngineProvider(load_config())
+    await provider.start()
+    try:
+        with pytest.raises(BlockedURLError):
+            await provider.fetch("http://169.254.169.254/")
+    finally:
+        await provider.aclose()

groundhog_mcp-0.1.0/tests/test_extract.py

@@ -0,0 +1,40 @@
+from groundhog_mcp.extract import to_markdown, truncate
+
+ARTICLE = """
+<html><head><title>T</title></head><body>
+<article><h1>Hello World</h1>
+<p>This is the first paragraph of a real article with enough words to be
+extracted by the content extractor. It mentions grounding for agents.</p>
+<p>This is a second substantial paragraph so the extractor crosses its
+content threshold and returns the main text rather than nothing at all.</p>
+</article></body></html>
+"""
+
+
+def test_truncate_under_limit():
+    text, cut = truncate("hello", 1000)
+    assert text == "hello"
+    assert cut is False
+
+
+def test_truncate_over_limit_marks_and_cuts():
+    text, cut = truncate("para one\n\npara two\n\npara three", 2)  # limit=8, no \n\n below it
+    assert cut is True
+    assert text.endswith("\n\n[... truncated]")
+
+
+def test_truncate_cuts_at_paragraph_boundary():
+    text, cut = truncate("para one\n\npara two\n\npara three", 3)  # limit=12, \n\n at index 8
+    assert cut is True
+    assert text == "para one\n\n[... truncated]"
+
+
+def test_to_markdown_extracts_article():
+    md, cut = to_markdown(ARTICLE, "", "https://example.com/a", 10000)
+    assert "Hello World" in md
+    assert cut is False
+
+
+def test_to_markdown_falls_back_to_text():
+    md, cut = to_markdown("<html><body></body></html>", "FALLBACK TEXT", "https://x.com", 10000)
+    assert "FALLBACK TEXT" in md

groundhog_mcp-0.1.0/tests/test_ratelimit.py

@@ -0,0 +1,48 @@
+import asyncio
+
+from groundhog_mcp.ratelimit import RateLimiter
+
+
+def make_clock(start=100.0):
+    state = {"t": start}
+    return state
+
+async def test_same_key_waits_once():
+    clock = make_clock()
+    sleeps = []
+
+    async def fake_sleep(d):
+        sleeps.append(d)
+        clock["t"] += d
+
+    rl = RateLimiter(5.0, clock=lambda: clock["t"], sleep=fake_sleep)
+    await rl.acquire("example.com")   # first call: no wait
+    await rl.acquire("example.com")   # second call: must wait 5s
+    assert sleeps == [5.0]
+
+
+async def test_different_keys_do_not_wait():
+    clock = make_clock()
+    sleeps = []
+
+    async def fake_sleep(d):
+        sleeps.append(d)
+        clock["t"] += d
+
+    rl = RateLimiter(5.0, clock=lambda: clock["t"], sleep=fake_sleep)
+    await rl.acquire("a.com")
+    await rl.acquire("b.com")
+    assert sleeps == []
+
+
+async def test_concurrent_same_key_serialized():
+    clock = make_clock()
+    sleeps = []
+
+    async def fake_sleep(d):
+        sleeps.append(d)
+        clock["t"] += d
+
+    rl = RateLimiter(5.0, clock=lambda: clock["t"], sleep=fake_sleep)
+    await asyncio.gather(rl.acquire("x.com"), rl.acquire("x.com"))
+    assert len(sleeps) == 1

groundhog_mcp-0.1.0/tests/test_read_url.py

@@ -0,0 +1,9 @@
+import pytest
+
+from groundhog_mcp.tools.read_url import read_url
+
+
+async def test_read_url_rejects_unknown_format():
+    # Validation happens at the boundary, before any engine call.
+    with pytest.raises(ValueError):
+        await read_url("https://example.com/", format="json")