web-intel-mcp 0.1.0__py3-none-any.whl

web_intel_mcp/__init__.py

@@ -0,0 +1,3 @@
+"""Web Intel: a pay-per-call MCP server for web content, SEO and tech-stack intelligence."""
+
+__version__ = "0.1.0"

web_intel_mcp/__main__.py

@@ -0,0 +1,3 @@
+from .server import main
+
+main()

web_intel_mcp/analyze.py

@@ -0,0 +1,186 @@
+"""Pure HTML analysis (no network): reader markdown, metadata, links, SEO audit, tech stack."""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Optional
+from urllib.parse import urljoin, urlsplit
+
+from bs4 import BeautifulSoup, Tag
+from markdownify import markdownify
+
+from .fetch import Page
+
+_NOISE = ["script", "style", "noscript", "template", "svg", "iframe", "form", "nav", "header", "footer", "aside"]
+_NOISE_HINT = re.compile(r"(cookie|consent|banner|newsletter|subscribe|share|social|sidebar|breadcrumb|menu|popup|modal|advert|promo)", re.I)
+
+
+def soup_of(page: Page) -> BeautifulSoup:
+    return BeautifulSoup(page.text, "html.parser")
+
+
+def _meta(soup: BeautifulSoup, *names: str) -> Optional[str]:
+    for n in names:
+        tag = soup.find("meta", attrs={"name": n}) or soup.find("meta", attrs={"property": n})
+        if isinstance(tag, Tag) and tag.get("content"):
+            return str(tag["content"]).strip()
+    return None
+
+
+def _main_node(soup: BeautifulSoup) -> Tag:
+    for sel in ("main", "article", "[role=main]", "#content", ".content", ".post", ".entry-content"):
+        node = soup.select_one(sel)
+        if node and len(node.get_text(strip=True)) > 200:
+            return node
+    return soup.body or soup
+
+
+def to_markdown(page: Page, max_chars: int = 20000) -> dict[str, Any]:
+    soup = soup_of(page)
+    title = (soup.title.get_text(strip=True) if soup.title else None) or _meta(soup, "og:title")
+    for t in soup(_NOISE):
+        t.decompose()
+    for t in soup.find_all(attrs={"class": _NOISE_HINT}) + soup.find_all(attrs={"id": _NOISE_HINT}):
+        if isinstance(t, Tag) and t.name not in ("body", "main", "article", "html"):
+            t.decompose()
+    node = _main_node(soup)
+    for a in node.find_all("a", href=True):
+        a["href"] = urljoin(page.final_url, str(a["href"]))
+    for img in node.find_all("img", src=True):
+        img["src"] = urljoin(page.final_url, str(img["src"]))
+    md = markdownify(str(node), heading_style="ATX", bullets="-", strip=["span"])
+    md = re.sub(r"\n{3,}", "\n\n", md).strip()
+    words = len(md.split())
+    truncated = len(md) > max_chars
+    return {"url": page.final_url, "title": title, "word_count": words,
+            "truncated": truncated or page.truncated, "markdown": md[:max_chars]}
+
+
+def _jsonld_types(soup: BeautifulSoup) -> list[str]:
+    out: list[str] = []
+    for s in soup.find_all("script", attrs={"type": "application/ld+json"}):
+        try:
+            data = json.loads(s.string or "")
+        except (ValueError, TypeError):
+            continue
+        stack = data if isinstance(data, list) else [data]
+        while stack:
+            item = stack.pop()
+            if isinstance(item, dict):
+                t = item.get("@type")
+                out.extend(t if isinstance(t, list) else [t] if t else [])
+                stack.extend(item.get("@graph") or [])
+    return sorted({str(t) for t in out})
+
+
+def metadata(page: Page) -> dict[str, Any]:
+    soup = soup_of(page)
+    canon = soup.find("link", rel="canonical")
+    html = soup.find("html")
+    headings = {f"h{i}": [h.get_text(" ", strip=True)[:160] for h in soup.find_all(f"h{i}")][:20] for i in (1, 2, 3)}
+    imgs = soup.find_all("img")
+    return {
+        "url": page.final_url, "status": page.status, "elapsed_ms": page.elapsed_ms, "redirects": page.redirects,
+        "title": soup.title.get_text(strip=True) if soup.title else None,
+        "description": _meta(soup, "description"),
+        "canonical": canon.get("href") if isinstance(canon, Tag) else None,
+        "lang": html.get("lang") if isinstance(html, Tag) else None,
+        "robots": _meta(soup, "robots"),
+        "open_graph": {k: _meta(soup, f"og:{k}") for k in ("title", "description", "image", "type", "url")},
+        "twitter_card": _meta(soup, "twitter:card"),
+        "jsonld_types": _jsonld_types(soup),
+        "headings": headings,
+        "images": {"total": len(imgs), "missing_alt": sum(1 for i in imgs if not (i.get("alt") or "").strip())},
+        "word_count": len(soup.get_text(" ", strip=True).split()),
+    }
+
+
+def links(page: Page, limit: int = 200) -> dict[str, Any]:
+    soup = soup_of(page)
+    host = urlsplit(page.final_url).hostname
+    internal: dict[str, str] = {}
+    external: dict[str, str] = {}
+    for a in soup.find_all("a", href=True):
+        href = str(a["href"]).strip()
+        if href.startswith(("#", "mailto:", "tel:", "javascript:")):
+            continue
+        absu = urljoin(page.final_url, href).split("#")[0]
+        bucket = internal if urlsplit(absu).hostname == host else external
+        bucket.setdefault(absu, a.get_text(" ", strip=True)[:120])
+    fmt = lambda d: [{"url": u, "text": t} for u, t in list(d.items())[:limit]]  # noqa: E731
+    return {"url": page.final_url, "internal_count": len(internal), "external_count": len(external),
+            "internal": fmt(internal), "external": fmt(external)}
+
+
+def seo_audit(page: Page) -> dict[str, Any]:
+    m = metadata(page)
+    checks: list[dict[str, Any]] = []
+
+    def add(cid: str, ok: bool, weight: int, fix: str, value: Any = None) -> None:
+        checks.append({"id": cid, "pass": ok, "weight": weight, "value": value, "fix": None if ok else fix})
+
+    title, desc = m["title"] or "", m["description"] or ""
+    add("status_200", m["status"] == 200, 10, "Serve the page with HTTP 200.", m["status"])
+    add("https", page.final_url.startswith("https://"), 8, "Serve over HTTPS and redirect HTTP to it.")
+    add("title_present", bool(title), 10, "Add a <title>.", title)
+    add("title_length", 30 <= len(title) <= 60, 5, "Keep the title between 30 and 60 characters.", len(title))
+    add("description_present", bool(desc), 8, "Add <meta name=description>.", desc[:160] or None)
+    add("description_length", 70 <= len(desc) <= 160, 4, "Keep the description between 70 and 160 characters.", len(desc))
+    add("single_h1", len(m["headings"]["h1"]) == 1, 7, "Use exactly one <h1>.", len(m["headings"]["h1"]))
+    add("canonical", bool(m["canonical"]), 5, "Add <link rel=canonical>.", m["canonical"])
+    add("lang", bool(m["lang"]), 3, "Set <html lang>.", m["lang"])
+    add("indexable", "noindex" not in (m["robots"] or "").lower()
+        and "noindex" not in page.headers.get("x-robots-tag", "").lower(), 10, "Remove noindex.", m["robots"])
+    add("open_graph", bool(m["open_graph"]["title"] and m["open_graph"]["image"]), 5,
+        "Add og:title and og:image for link previews.")
+    add("structured_data", bool(m["jsonld_types"]), 6, "Add JSON-LD (Organization, Product, Article, FAQPage...).",
+        m["jsonld_types"])
+    add("img_alt", m["images"]["missing_alt"] == 0, 4, "Add alt text to every meaningful image.", m["images"])
+    add("content_depth", m["word_count"] >= 300, 6, "Thin content: aim for 300+ words of real text.", m["word_count"])
+    add("viewport", bool(re.search(r'<meta[^>]+name=["\']viewport', page.text, re.I)), 5, "Add a viewport meta tag.")
+    add("fast_response", m["elapsed_ms"] < 1500, 4, "Server response is slow (>1.5 s).", m["elapsed_ms"])
+    total = sum(c["weight"] for c in checks)
+    score = round(100 * sum(c["weight"] for c in checks if c["pass"]) / total)
+    fixes = sorted((c for c in checks if not c["pass"]), key=lambda c: -c["weight"])
+    grade = next(g for g, floor in (("A", 90), ("B", 80), ("C", 70), ("D", 60), ("F", 0)) if score >= floor)
+    return {"url": page.final_url, "score": score, "grade": grade, "top_fixes": [c["fix"] for c in fixes[:5]], "checks": checks}
+
+
+_SIGNATURES: list[tuple[str, str, str]] = [
+    # (name, where: html|header, regex)
+    ("Next.js", "html", r"/_next/|__NEXT_DATA__"), ("Nuxt", "html", r"/_nuxt/|__NUXT__"),
+    ("React", "html", r"data-reactroot|react-dom"), ("Vue", "html", r"data-v-[0-9a-f]{6}|vue(\.min)?\.js"),
+    ("Svelte", "html", r"svelte-[a-z0-9]{5,}"), ("Astro", "html", r"astro-island|/_astro/"),
+    ("Angular", "html", r"ng-version="), ("Gatsby", "html", r"___gatsby"), ("Remix", "html", r"__remixContext"),
+    ("WordPress", "html", r"wp-content|wp-includes"), ("Shopify", "html", r"cdn\.shopify\.com|Shopify\.theme"),
+    ("Webflow", "html", r"webflow\.(js|css)|data-wf-page"), ("Wix", "html", r"static\.wixstatic\.com"),
+    ("Squarespace", "html", r"squarespace\.com"), ("Framer", "html", r"framerusercontent\.com"),
+    ("Ghost", "html", r'content="Ghost'), ("Tailwind CSS", "html", r'class="[^"]*\b(?:flex|grid) [^"]*\b(?:px|py|mt|mb)-\d'),
+    ("Bootstrap", "html", r"bootstrap(\.min)?\.(css|js)"), ("jQuery", "html", r"jquery(\.min)?\.js"),
+    ("Google Analytics", "html", r"googletagmanager\.com/gtag|google-analytics\.com"),
+    ("Google Tag Manager", "html", r"googletagmanager\.com/gtm\.js"), ("Plausible", "html", r"plausible\.io/js"),
+    ("PostHog", "html", r"posthog"), ("Segment", "html", r"cdn\.segment\.com"), ("Hotjar", "html", r"hotjar\.com"),
+    ("Intercom", "html", r"widget\.intercom\.io"), ("Crisp", "html", r"client\.crisp\.chat"),
+    ("Stripe", "html", r"js\.stripe\.com"), ("Lemon Squeezy", "html", r"lemonsqueezy"),
+    ("Cloudflare", "header", r"^server: cloudflare|^cf-ray:"), ("Vercel", "header", r"^server: vercel|^x-vercel-id:"),
+    ("Netlify", "header", r"^server: netlify|^x-nf-request-id:"), ("Fly.io", "header", r"^fly-request-id:"),
+    ("nginx", "header", r"^server: nginx"), ("Apache", "header", r"^server: apache"),
+    ("AWS CloudFront", "header", r"^x-amz-cf-id:|^via: .*cloudfront"), ("PHP", "header", r"^x-powered-by: php"),
+    ("Express", "header", r"^x-powered-by: express"),
+]
+
+
+def tech_stack(page: Page) -> dict[str, Any]:
+    hdr = "\n".join(f"{k}: {v}" for k, v in page.headers.items()).lower()
+    found = []
+    for name, where, rx in _SIGNATURES:
+        hay = page.text if where == "html" else hdr
+        if re.search(rx, hay, re.I | re.M):
+            found.append({"name": name, "evidence": where})
+    gen = re.search(r'<meta[^>]+name=["\']generator["\'][^>]+content=["\']([^"\']+)', page.text, re.I)
+    sec = {h: page.headers.get(h) is not None for h in
+           ("strict-transport-security", "content-security-policy", "x-frame-options", "x-content-type-options",
+            "referrer-policy")}
+    return {"url": page.final_url, "technologies": found, "generator": gen.group(1) if gen else None,
+            "security_headers": sec, "server": page.headers.get("server")}

web_intel_mcp/billing.py

@@ -0,0 +1,167 @@
+"""Per-call credit billing on top of keyflint_verify.mcp.KeyFlint.
+
+stdio: the license comes from KEYFLINT_LICENSE (one buyer per process).
+HTTP:  the license comes from each request (``Authorization: Bearer LIC-...`` or
+``X-KeyFlint-License``), so one hosted server bills many buyers.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import pathlib
+import threading
+import uuid
+from collections import OrderedDict
+from contextlib import asynccontextmanager
+from typing import Any, AsyncIterator, Mapping, Optional
+
+from keyflint_verify.mcp import LICENSE_INVALID, KeyFlint, LicenseError
+
+LICENSE_HEADERS = ("x-license-key", "authorization")
+_MAX_CLIENTS = 1024
+
+DEFAULT_GATEWAY = "https://licenses.keyflint.dev"
+DEFAULT_CHECKOUT = "https://mcp.keyflint.dev/precios/"
+_KEYRING_TTL = 6 * 3600
+
+
+def _env(name: str) -> str:
+    return (os.environ.get("KEYFLINT_" + name) or os.environ.get("KEYFORGE_" + name) or "").strip()
+
+
+def _fetch_keyring(gateway: str) -> str:
+    """Gateway public keyring, cached in ~/.keyflint (stale copy used when offline)."""
+    import hashlib
+    import json
+    import time
+    import urllib.request
+
+    cache = pathlib.Path(os.environ.get("KEYFLINT_KEYRING_CACHE")
+                         or pathlib.Path.home() / ".keyflint" / "keyring.json")
+    slot = hashlib.sha256(gateway.encode()).hexdigest()[:16]
+    try:
+        stored = json.loads(cache.read_text("utf-8"))
+    except (OSError, ValueError):
+        stored = {}
+    entry = stored.get(slot) or {}
+    if entry.get("keyring") and time.time() - float(entry.get("at", 0)) < _KEYRING_TTL:
+        return str(entry["keyring"])
+    try:
+        req = urllib.request.Request(gateway + "/api/v1/licenses/public-key",
+                                     headers={"Accept": "application/json"})
+        with urllib.request.urlopen(req, timeout=4) as resp:  # noqa: S310 - https gateway
+            keyring = str(json.loads(resp.read().decode("utf-8"))["data"]["keyring"]).strip()
+    except Exception:  # noqa: BLE001 - offline: fall back to the cached keyring
+        return str(entry.get("keyring") or "")
+    if keyring:
+        try:
+            stored[slot] = {"keyring": keyring, "at": time.time()}
+            cache.parent.mkdir(parents=True, exist_ok=True)
+            cache.write_text(json.dumps(stored), "utf-8")
+        except OSError:
+            pass
+    return keyring
+
+
+def apply_defaults() -> None:
+    """Only KEYFLINT_LICENSE is required: gateway, keyring and checkout default to KeyFlint's."""
+    if not _env("URL"):
+        os.environ["KEYFLINT_URL"] = DEFAULT_GATEWAY
+    if not _env("CHECKOUT_URL"):
+        os.environ["KEYFLINT_CHECKOUT_URL"] = DEFAULT_CHECKOUT
+    if not (_env("PUBLIC_KEYS") or _env("PUBLIC_KEY")):
+        keyring = _fetch_keyring(_env("URL").rstrip("/"))
+        if keyring:
+            os.environ["KEYFLINT_PUBLIC_KEYS"] = keyring
+
+
+def license_from_headers(headers: Optional[Mapping[str, str]]) -> Optional[str]:
+    if not headers:
+        return None
+    lower = {k.lower(): v for k, v in headers.items()}
+    raw = lower.get("x-license-key") or ""
+    if not raw:
+        auth = lower.get("authorization") or ""
+        if auth[:7].lower() == "bearer ":
+            raw = auth[7:]
+    raw = raw.strip()
+    return raw or None
+
+
+class Billing:
+    """Resolves the buyer's license and debits credits around a tool body."""
+
+    def __init__(self, *, hosted: bool = False, **kf_kwargs: Any) -> None:
+        self.hosted = hosted
+        if "public_keys" not in kf_kwargs:
+            apply_defaults()
+        self._kf_kwargs = kf_kwargs
+        self._env_kf = KeyFlint(**kf_kwargs)
+        self._clients: "OrderedDict[str, KeyFlint]" = OrderedDict()
+        self._lock = threading.Lock()
+
+    @property
+    def checkout_url(self) -> Optional[str]:
+        return self._env_kf.checkout_url
+
+    def client(self, headers: Optional[Mapping[str, str]] = None) -> KeyFlint:
+        key = license_from_headers(headers)
+        if key is None:
+            if self.hosted:
+                text = "Send your KeyFlint license as 'Authorization: Bearer LIC-...'."
+                if self.checkout_url:
+                    text += f" Get one: {self.checkout_url}"
+                raise LicenseError(LICENSE_INVALID, text, {"reason": "ERR_EMPTY_TOKEN",
+                                                           "checkout_url": self.checkout_url})
+            return self._env_kf
+        with self._lock:
+            kf = self._clients.get(key)
+            if kf is None:
+                # Hosted: server clock is trusted, no node lock (the buyer's machine is not ours).
+                kf = KeyFlint(license_key=key, machine_id=None, clock_guard=False,
+                              public_keys=self._env_kf.public_keys, gateway_url=self._env_kf.gateway_url,
+                              checkout_url=self._env_kf.checkout_url, timeout=self._env_kf.timeout)
+                self._clients[key] = kf
+                if len(self._clients) > _MAX_CLIENTS:
+                    self._clients.popitem(last=False)
+            else:
+                self._clients.move_to_end(key)
+            return kf
+
+    @asynccontextmanager
+    async def charge(self, credits: int, tool: str, headers: Optional[Mapping[str, str]] = None,
+                     features: Optional[list[str]] = None) -> AsyncIterator[dict[str, Any]]:
+        """Offline license check, debit, run body; refund if the body raises."""
+        kf = self.client(headers)
+        kf.check(features)
+        idem = f"wi-{uuid.uuid4().hex}"
+        receipt = await asyncio.to_thread(kf.deduct, credits, idempotency_key=idem, event_type=f"mcp:{tool}")
+        try:
+            yield receipt
+        except BaseException:
+            await asyncio.to_thread(kf.refund, idem, "tool_error")
+            raise
+
+    async def balance(self, headers: Optional[Mapping[str, str]] = None) -> dict[str, Any]:
+        import json
+        import urllib.parse
+        import urllib.request
+
+        kf = self.client(headers)
+        kf.check()
+        if not kf.gateway_url:
+            raise LicenseError(-32063, "KEYFLINT_URL is not set.", {"reason": "GATEWAY_NOT_CONFIGURED"})
+        url = f"{kf.gateway_url}/api/v1/quota/balance?" + urllib.parse.urlencode({"license_key": kf.license_key})
+
+        def _get() -> dict[str, Any]:
+            req = urllib.request.Request(url, headers={"Accept": "application/json"})
+            with urllib.request.urlopen(req, timeout=kf.timeout) as resp:  # noqa: S310 - configured URL
+                data = json.loads(resp.read().decode("utf-8") or "{}")
+                return dict(data.get("data", data))
+
+        return await asyncio.to_thread(_get)
+
+
+def from_env() -> Billing:
+    return Billing(hosted=os.environ.get("WEB_INTEL_TRANSPORT", "stdio") != "stdio")

web_intel_mcp/fetch.py

@@ -0,0 +1,122 @@
+"""Safe page fetching: http(s) only, no private networks, bounded size, manual redirects."""
+
+from __future__ import annotations
+
+import asyncio
+import codecs
+import ipaddress
+import os
+import socket
+import time
+from dataclasses import dataclass, field
+from urllib.parse import urljoin, urlsplit
+
+import httpx
+from mcp.server.mcpserver.exceptions import ToolError
+
+MAX_BYTES = int(os.environ.get("WEB_INTEL_MAX_BYTES", 3_000_000))
+TIMEOUT = float(os.environ.get("WEB_INTEL_TIMEOUT", 15))
+TOTAL_TIMEOUT = float(os.environ.get("WEB_INTEL_TOTAL_TIMEOUT", 30))  # whole fetch, incl. redirects
+MAX_REDIRECTS = 5
+USER_AGENT = os.environ.get("WEB_INTEL_USER_AGENT", "WebIntelMCP/0.1")
+
+
+class FetchError(ToolError):
+    """Bad input or unreachable page; the client sees the message as an isError result."""
+
+
+@dataclass
+class Page:
+    url: str
+    final_url: str
+    status: int
+    headers: dict[str, str]
+    text: str
+    elapsed_ms: int
+    redirects: list[str] = field(default_factory=list)
+    truncated: bool = False
+
+
+def _allow_private() -> bool:
+    return os.environ.get("WEB_INTEL_ALLOW_PRIVATE") == "1"
+
+
+def validate_url(url: str) -> str:
+    url = (url or "").strip()
+    if "://" not in url:
+        url = "https://" + url
+    parts = urlsplit(url)
+    if parts.scheme not in ("http", "https") or not parts.hostname:
+        raise FetchError(f"Only http(s) URLs are supported: {url!r}")
+    return url
+
+
+def _check_peer(resp: httpx.Response) -> None:
+    """Re-check the address actually connected to (defeats DNS rebinding between check and connect)."""
+    if _allow_private():
+        return
+    stream = resp.extensions.get("network_stream")
+    addr = stream.get_extra_info("server_addr") if stream is not None else None
+    if addr and not ipaddress.ip_address(addr[0]).is_global:
+        raise FetchError("The host resolved to a non-public address on connect; refused.")
+
+
+async def _check_host(host: str) -> None:
+    if _allow_private():
+        return
+    try:
+        infos = await asyncio.to_thread(socket.getaddrinfo, host, None)
+    except socket.gaierror as exc:
+        raise FetchError(f"Cannot resolve host {host!r}.") from exc
+    for info in infos:
+        ip = ipaddress.ip_address(info[4][0])
+        if not ip.is_global:
+            raise FetchError(f"Host {host!r} resolves to a non-public address; refused.")
+
+
+def _encoding(name: str | None) -> str:
+    try:
+        return codecs.lookup(name or "utf-8").name
+    except LookupError:
+        return "utf-8"
+
+
+async def fetch(url: str) -> Page:
+    """Fetch with a hard deadline, so a server that drips bytes cannot hold a worker forever."""
+    url = validate_url(url)
+    try:
+        return await asyncio.wait_for(_fetch(url), TOTAL_TIMEOUT)
+    except asyncio.TimeoutError as exc:
+        raise FetchError(f"{url} took longer than {TOTAL_TIMEOUT:g}s; gave up.") from exc
+
+
+async def _fetch(url: str) -> Page:
+    current, redirects = url, []
+    start = time.perf_counter()
+    async with httpx.AsyncClient(timeout=TIMEOUT, follow_redirects=False,
+                                 headers={"User-Agent": USER_AGENT, "Accept": "text/html,*/*;q=0.8"}) as client:
+        for _ in range(MAX_REDIRECTS + 1):
+            await _check_host(urlsplit(current).hostname or "")
+            try:
+                async with client.stream("GET", current) as resp:
+                    _check_peer(resp)
+                    if resp.is_redirect and "location" in resp.headers:
+                        redirects.append(current)
+                        current = validate_url(urljoin(current, resp.headers["location"]))
+                        continue
+                    buf, truncated = bytearray(), False
+                    async for chunk in resp.aiter_bytes():
+                        buf.extend(chunk)
+                        if len(buf) > MAX_BYTES:
+                            truncated = True
+                            del buf[MAX_BYTES:]
+                            break
+                    enc = _encoding(resp.charset_encoding)
+                    return Page(url=url, final_url=current, status=resp.status_code,
+                                headers={k.lower(): v for k, v in resp.headers.items()},
+                                text=buf.decode(enc, errors="replace"),
+                                elapsed_ms=int((time.perf_counter() - start) * 1000),
+                                redirects=redirects, truncated=truncated)
+            except httpx.HTTPError as exc:
+                raise FetchError(f"Could not fetch {current}: {exc.__class__.__name__}: {exc}") from exc
+    raise FetchError(f"Too many redirects (>{MAX_REDIRECTS}) starting at {url}.")

web_intel_mcp/server.py

@@ -0,0 +1,181 @@
+"""Web Intel MCP: web pages -> clean markdown, metadata, links, SEO audit, tech stack.
+
+Every paid tool debits KeyFlint credits before it fetches and refunds them if the fetch fails.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+from typing import Any
+
+from mcp.server.mcpserver import Context, MCPServer
+from mcp.types import ToolAnnotations
+
+from . import analyze
+from .billing import Billing
+from .fetch import FetchError, fetch, validate_url
+
+DEFAULT_PRICES = {"read_url": 1, "page_metadata": 1, "extract_links": 1, "tech_stack": 2, "seo_audit": 3}
+MAX_COMPARE = 5
+
+
+def _prices() -> dict[str, int]:
+    prices = dict(DEFAULT_PRICES)
+    for item in filter(None, (p.strip() for p in os.environ.get("WEB_INTEL_PRICES", "").split(","))):
+        name, _, n = item.partition("=")
+        if name.strip() in prices and n.strip().isdigit() and int(n) > 0:
+            prices[name.strip()] = int(n)
+    return prices
+
+
+PRICES = _prices()
+billing = Billing(hosted=os.environ.get("WEB_INTEL_TRANSPORT", "stdio") != "stdio")
+RO = ToolAnnotations(readOnlyHint=True, openWorldHint=True, idempotentHint=True)
+
+mcp = MCPServer(
+    "web-intel-mcp",
+    title="Web Intel",
+    version="0.1.0",
+    website_url="https://github.com/ArceeDev/web-intel-mcp",
+    instructions=(
+        "Turns web pages into agent-ready data. Paid tools cost KeyFlint credits (see `pricing`); "
+        "a failed fetch is refunded automatically. Use `read_url` for content, `page_metadata` for SEO tags, "
+        "`seo_audit` for a scored checklist with fixes, `tech_stack` to identify frameworks and hosting."
+    ),
+)
+
+
+def _headers(ctx: Context) -> Any:
+    try:
+        return ctx.headers
+    except ValueError:
+        return None
+
+
+def _ok(page: analyze.Page) -> analyze.Page:
+    if page.status >= 400:
+        raise FetchError(f"{page.final_url} returned HTTP {page.status}; credits refunded.")
+    return page
+
+
+def _run(fn: Any, *args: Any) -> Any:
+    """Analysis guard: pathological HTML becomes a clean, refunded error instead of a crash."""
+    try:
+        return fn(*args)
+    except RecursionError as exc:
+        raise FetchError("The page HTML is nested too deeply to parse; credits refunded.") from exc
+
+
+async def _paid(ctx: Context, tool: str, url: str, fn: Any, *args: Any) -> tuple[dict, dict]:
+    url = validate_url(url)  # bad input is rejected before any debit
+    async with billing.charge(PRICES[tool], tool, _headers(ctx)) as receipt:
+        out = _run(fn, _ok(await fetch(url)), *args)
+    return out, receipt
+
+
+def _billing_note(receipt: dict, credits: int) -> dict[str, Any]:
+    return {"credits_charged": credits, "credits_remaining": receipt.get("remaining_balance")}
+
+
+@mcp.tool(annotations=RO)
+async def read_url(url: str, ctx: Context, max_chars: int = 20000) -> dict[str, Any]:
+    """Fetch a web page and return its main content as clean Markdown (nav, ads and cookie banners removed).
+
+    Args:
+        url: Page URL (https:// is assumed when no scheme is given).
+        max_chars: Cap on returned markdown length (default 20000, max 100000).
+    """
+    max_chars = max(500, min(int(max_chars), 100_000))
+    url = validate_url(url)
+    out, receipt = await _paid(ctx, "read_url", url, analyze.to_markdown, max_chars)
+    return {**out, "billing": _billing_note(receipt, PRICES["read_url"])}
+
+
+@mcp.tool(annotations=RO)
+async def page_metadata(url: str, ctx: Context) -> dict[str, Any]:
+    """Title, description, canonical, lang, robots, Open Graph, JSON-LD types, H1-H3 and image alt stats."""
+    out, receipt = await _paid(ctx, "page_metadata", url, analyze.metadata)
+    return {**out, "billing": _billing_note(receipt, PRICES["page_metadata"])}
+
+
+@mcp.tool(annotations=RO)
+async def extract_links(url: str, ctx: Context, limit: int = 200) -> dict[str, Any]:
+    """Internal and external links on a page (deduplicated, absolute URLs, anchor text)."""
+    out, receipt = await _paid(ctx, "extract_links", url, analyze.links, max(1, min(int(limit), 1000)))
+    return {**out, "billing": _billing_note(receipt, PRICES["extract_links"])}
+
+
+@mcp.tool(annotations=RO)
+async def tech_stack(url: str, ctx: Context) -> dict[str, Any]:
+    """Detect frameworks, CMS, analytics, payments, CDN/hosting and security headers of a site."""
+    out, receipt = await _paid(ctx, "tech_stack", url, analyze.tech_stack)
+    return {**out, "billing": _billing_note(receipt, PRICES["tech_stack"])}
+
+
+@mcp.tool(annotations=RO)
+async def seo_audit(url: str, ctx: Context) -> dict[str, Any]:
+    """Scored on-page SEO audit (0-100, A-F) with 16 weighted checks and the top fixes to apply first."""
+    out, receipt = await _paid(ctx, "seo_audit", url, analyze.seo_audit)
+    return {**out, "billing": _billing_note(receipt, PRICES["seo_audit"])}
+
+
+@mcp.tool(annotations=RO)
+async def compare_seo(urls: list[str], ctx: Context) -> dict[str, Any]:
+    """Audit 2-5 pages (e.g. you vs competitors) and rank them. Costs seo_audit price x number of URLs.
+
+    Unreachable URLs are reported, not billed: their share is refunded as one batch.
+    """
+    clean = list(dict.fromkeys(validate_url(u) for u in urls))
+    if not 2 <= len(clean) <= MAX_COMPARE:
+        raise FetchError(f"Pass between 2 and {MAX_COMPARE} distinct URLs.")
+    billing.client(_headers(ctx)).check()  # no fetching for callers without a valid license
+    unit = PRICES["seo_audit"]
+    results: list[dict[str, Any]] = []
+    failed: list[dict[str, str]] = []
+    # Debit only the URLs that actually fetched, so failures never cost anything.
+    pages = []
+    for u in clean:
+        try:
+            pages.append(_ok(await fetch(u)))
+        except FetchError as exc:
+            failed.append({"url": u, "error": str(exc)})
+    if not pages:
+        raise FetchError("None of the URLs could be fetched; nothing was charged.")
+    async with billing.charge(unit * len(pages), "compare_seo", _headers(ctx)) as receipt:
+        for p in pages:
+            a = _run(analyze.seo_audit, p)
+            results.append({"url": a["url"], "score": a["score"], "grade": a["grade"], "top_fixes": a["top_fixes"]})
+    results.sort(key=lambda r: -r["score"])
+    return {"ranking": results, "failed": failed, "billing": _billing_note(receipt, unit * len(pages))}
+
+
+@mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, openWorldHint=False))
+def pricing() -> dict[str, Any]:
+    """Credit price of each tool and where to buy credits. Free."""
+    return {"credits_per_call": {**PRICES, "compare_seo": f"{PRICES['seo_audit']} x URLs"},
+            "refunds": "Failed fetches are refunded automatically.",
+            "buy_credits": billing.checkout_url or "Set KEYFLINT_CHECKOUT_URL"}
+
+
+@mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, openWorldHint=False))
+async def credits_balance(ctx: Context) -> dict[str, Any]:
+    """Remaining credits on your KeyFlint license. Free."""
+    return await billing.balance(_headers(ctx))
+
+
+def main() -> None:
+    p = argparse.ArgumentParser(prog="web-intel-mcp", description=__doc__)
+    p.add_argument("--http", action="store_true", help="Serve Streamable HTTP (hosted, license per request).")
+    p.add_argument("--host", default=os.environ.get("HOST", "0.0.0.0"))
+    p.add_argument("--port", type=int, default=int(os.environ.get("PORT", 8080)))
+    args = p.parse_args()
+    if args.http:
+        billing.hosted = True
+        mcp.run("streamable-http", host=args.host, port=args.port, stateless_http=True, json_response=True)
+    else:
+        mcp.run("stdio")
+
+
+if __name__ == "__main__":
+    main()

web_intel_mcp-0.1.0.dist-info/METADATA

@@ -0,0 +1,108 @@
+Metadata-Version: 2.4
+Name: web-intel-mcp
+Version: 0.1.0
+Summary: Metered MCP server: web pages to clean Markdown, metadata, links, SEO audits and tech stack. Billed per call with KeyFlint credits.
+Author: ArceeDev
+License-Expression: BUSL-1.1
+Project-URL: Homepage, https://github.com/ArceeDev/web-intel-mcp
+Project-URL: Source, https://github.com/ArceeDev/web-intel-mcp
+Keywords: mcp,seo,markdown,scraping,agents
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: mcp>=2.0
+Requires-Dist: keyflint-verify[fast]>=1.2.0
+Requires-Dist: httpx>=0.27
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: markdownify>=0.13
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == "test"
+Requires-Dist: pytest-asyncio>=0.23; extra == "test"
+Dynamic: license-file
+
+# Web Intel MCP
+
+Agent-ready web intelligence as a pay-per-call MCP server. Licensing and credits are handled by
+[KeyFlint](https://keyflint.dev) (offline Ed25519 license check + credit ledger), used here only as the billing backend. Failed fetches are refunded
+automatically; invalid input is rejected before any debit.
+
+| Tool | Credits | Returns |
+|---|---|---|
+| `read_url` | 1 | Main content as clean Markdown (nav, footer, cookie banners removed) |
+| `page_metadata` | 1 | Title, description, canonical, lang, robots, Open Graph, JSON-LD types, H1-H3, image alt stats |
+| `extract_links` | 1 | Deduplicated internal / external links with anchor text |
+| `tech_stack` | 2 | Frameworks, CMS, analytics, payments, CDN/hosting, security headers |
+| `seo_audit` | 3 | 0-100 score, A-F grade, 16 weighted checks, top 5 fixes |
+| `compare_seo` | 3 x URLs | Ranks 2-5 pages (you vs competitors); only fetched URLs are billed |
+| `pricing`, `credits_balance` | free | Price list, remaining credits |
+
+Prices are overridable: `WEB_INTEL_PRICES="seo_audit=5,read_url=2"`.
+
+## Install
+
+All you need is your KeyFlint license (`LIC-...`). The gateway URL, its public key and the checkout link have defaults.
+
+**Claude Desktop, one click:** download `web-intel-mcp-X.Y.Z.mcpb` from the [latest release](https://github.com/ArceeDev/web-intel-mcp/releases/latest), double-click it and paste your license. No terminal, no JSON.
+
+**Claude Code:**
+
+```bash
+claude mcp add web-intel -e KEYFLINT_LICENSE=LIC-... -- uvx web-intel-mcp
+```
+
+**Cursor, Windsurf, Claude Desktop (JSON):**
+
+```json
+{
+  "mcpServers": {
+    "web-intel": {
+      "command": "uvx",
+      "args": ["web-intel-mcp"],
+      "env": { "KEYFLINT_LICENSE": "LIC-..." }
+    }
+  }
+}
+```
+
+Optional: `KEYFLINT_URL`, `KEYFLINT_PUBLIC_KEYS` (fetched from the gateway and cached in `~/.keyflint/keyring.json` when unset), `KEYFLINT_CHECKOUT_URL`.
+
+## Hosted mode (sellers)
+
+Paid logic stays on your server. One process bills many buyers; each
+request carries the buyer's license.
+
+```bash
+docker build -t web-intel .
+docker run -p 8080:8080 \
+  web-intel
+```
+
+Buyers connect to `https://<host>/mcp` (Streamable HTTP, stateless) with
+`Authorization: Bearer LIC-...` (or `X-License-Key`). Missing or forged licenses get `-32060`.
+
+## Billing flow per call
+
+1. URL validated (http/https only). Bad input: error, nothing charged.
+2. Offline Ed25519 license check (`keyflint-verify`), cached 60 s.
+3. `POST /api/v1/quota/deduct` with a fresh idempotency key. Out of credits: `-32062` with the checkout link.
+4. Fetch + analysis. If anything fails: `POST /api/v1/quota/refund` (same key), the client gets `isError`.
+
+## Safety
+
+Requests to private, loopback and link-local addresses are refused (checked on every redirect hop),
+responses are capped at 3 MB (`WEB_INTEL_MAX_BYTES`), 15 s timeout (`WEB_INTEL_TIMEOUT`), max 5 redirects.
+`WEB_INTEL_ALLOW_PRIVATE=1` disables the address check (tests only).
+
+## Tests
+
+End-to-end with a fake KeyFlint quota gateway, locally signed licenses and a local test site
+(no network needed):
+
+```bash
+pip install -e ".[test]"
+python -m pytest -q
+```
+
+## License
+
+Business Source License 1.1. Source available; production use needs a KeyFlint license (or must be non-commercial). Each version becomes Apache 2.0 four years after release. See [LICENSE](LICENSE).

web_intel_mcp-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+web_intel_mcp/__init__.py,sha256=VdPXr3SLgseHuKFhoWLRajKOv_QRECUXN9fqnf_m_2A,116
+web_intel_mcp/__main__.py,sha256=3dYKHfmWsrdExFlTFlcR5a_icR9fAkn06Yh14TQkEd8,33
+web_intel_mcp/analyze.py,sha256=fYseJhB7-2HdSS8_9B18RIPaaX0Bkm_yAlB1_V-eGVY,10352
+web_intel_mcp/billing.py,sha256=OowbZvqPPZOiMc3dgU5J8JS5aiN9Ex4cSNck21WLoQs,6779
+web_intel_mcp/fetch.py,sha256=hZ-JUBON08k0jrUTLpWLfWTH5JROEpINeK1w0jYCJpA,4652
+web_intel_mcp/server.py,sha256=6vIA0dd3o3wRme9jCByN_TgDi2E6CLUHOaSAN_xH8jU,7528
+web_intel_mcp-0.1.0.dist-info/licenses/LICENSE,sha256=WmSvxxXOjcsfs8DUt4scvKEXcG0hKD6U6APjD-z3PLg,3548
+web_intel_mcp-0.1.0.dist-info/METADATA,sha256=P9kEI3HEDqQHTJfLwAgzz_RFIW7Qlg2D2fmeMh2_Hdc,4139
+web_intel_mcp-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+web_intel_mcp-0.1.0.dist-info/entry_points.txt,sha256=teSnQeGbxZI6B1NpmvhVQbbCYXn1ly8FCg5X7mkmnzU,60
+web_intel_mcp-0.1.0.dist-info/top_level.txt,sha256=P_ZhqQZMOIQj20n0_mxemBCr-jV8c9NuQNHaItTTHHc,14
+web_intel_mcp-0.1.0.dist-info/RECORD,,

web_intel_mcp-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

web_intel_mcp-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+web-intel-mcp = web_intel_mcp.server:main

web_intel_mcp-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,70 @@
+License text copyright (c) 2020 MariaDB Corporation Ab, All Rights Reserved.
+"Business Source License" is a trademark of MariaDB Corporation Ab.
+
+Parameters
+
+Licensor:             ArceeDev (KeyFlint, https://keyflint.dev, legal@keyflint.dev)
+Licensed Work:        Web Intel MCP (web-intel-mcp), version 0.1.0 or later.
+                      The Licensed Work is (c) 2026 ArceeDev.
+Additional Use Grant: You may make production use of the Licensed Work:
+                      (a) with a valid KeyFlint license for the Licensed Work
+                          obtained from the Licensor (https://mcp.keyflint.dev),
+                          with tool calls metered through the Licensor's
+                          KeyFlint gateway; or
+                      (b) for personal, educational, research or evaluation
+                          purposes that are not commercial.
+                      This grant does not cover removing, disabling or
+                      bypassing the license verification or credit metering,
+                      nor offering the Licensed Work, or a modified version of
+                      it, to third parties on a hosted, embedded or resold
+                      basis.
+Change Date:          2030-11-05
+Change License:       Apache License, Version 2.0
+
+For information about alternative licensing arrangements for the Licensed Work,
+please contact the Licensor at legal@keyflint.dev or https://keyflint.dev.
+
+Notice
+
+Business Source License 1.1
+
+Terms
+
+The Licensor hereby grants you the right to copy, modify, create derivative
+works, redistribute, and make non-production use of the Licensed Work. The
+Licensor may make an Additional Use Grant, above, permitting limited production use.
+
+Effective on the Change Date, or the fourth anniversary of the first publicly
+available distribution of a specific version of the Licensed Work under this
+License, whichever comes first, the Licensor hereby grants you rights under
+the terms of the Change License, and the rights granted in the paragraph
+above terminate.
+
+If your use of the Licensed Work does not comply with the requirements
+currently in effect as described in this License, you must purchase a
+commercial license from the Licensor, its affiliated entities, or authorized
+resellers, or you must refrain from using the Licensed Work.
+
+All copies of the original and modified Licensed Work, and derivative works
+of the Licensed Work, are subject to this License. This License applies
+separately for each version of the Licensed Work and the Change Date may vary
+for each version of the Licensed Work released by Licensor.
+
+You must conspicuously display this License on each original or modified copy
+of the Licensed Work. If you receive the Licensed Work in original or
+modified form from a third party, the terms and conditions set forth in this
+License apply to your use of that work.
+
+Any use of the Licensed Work in violation of this License will automatically
+terminate your rights under this License for the current and all other
+versions of the Licensed Work.
+
+This License does not grant you any right in any trademark or logo of
+Licensor or its affiliates (provided that you may use a trademark or logo of
+Licensor as expressly required by this License).
+
+TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
+AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
+EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
+TITLE.

web_intel_mcp-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+web_intel_mcp