sniffmcp-cli 0.4.1__py3-none-any.whl

sniffmcp/cli.py

@@ -0,0 +1,242 @@
+"""sniffmcp CLI. Same engine as the MCP server.
+
+Exit contract (stable):
+  0  scan completed, no active finding at/above --fail-on
+  1  scan completed, at least one active finding at/above threshold
+  2  target or configuration error
+  3  analysis incomplete (server unreachable / timed out)
+
+Usage:
+  sniffmcp scan '{"command":"npx","args":["-y","pkg@1.2.3"]}'
+  sniffmcp scan https://mcp.example.com/mcp
+  sniffmcp scan cfg.json --format sarif --output results.sarif
+  sniffmcp fleet                      # every server in your Claude/Cursor/Windsurf configs
+  sniffmcp watch cfg.json --name myserver --webhook https://hooks.slack.com/...
+  sniffmcp watch --name myserver --accept     # accept the current manifest as the new baseline
+  sniffmcp watch --run                        # check all watches on their intervals, forever
+  sniffmcp crawl run --output crawl-report.md # daily: registry sync, probe public servers, package history
+"""
+from __future__ import annotations
+import argparse, asyncio, json, os, sys
+from pathlib import Path
+
+from .client import ConnectError, transport_of
+from .engine import scan
+from .scoring import summarize
+from .report import to_sarif, fleet_to_sarif, to_json_report, diff_baseline, apply_suppressions, gate, SEVERITY_RANK
+from .state import Store
+from . import watcher
+from .fleet import scan_fleet, fleet_console
+
+
+def load_config(arg: str) -> dict:
+    """Inline JSON, a path to a JSON file, or an http(s) URL."""
+    arg = arg.strip()
+    if arg.startswith(("https://", "http://")):
+        return {"url": arg}
+    if arg.startswith("{"):
+        return json.loads(arg)
+    return json.loads(Path(arg).read_text())
+
+
+def console_print(report) -> None:
+    print(f"\n  {report.target}\n  score {report.score}/100  grade {report.grade}  ({report.tool_count} tools)")
+    s = summarize(report.findings)
+    print(f"  critical:{s['critical']} high:{s['high']} medium:{s['medium']} low:{s['low']} info:{s['info']}")
+    for f in sorted(report.findings, key=lambda f: -SEVERITY_RANK.get(f.severity, 0)):
+        mark = " (suppressed)" if f.suppressed else ""
+        print(f"  [{f.severity:8}] {f.check_id:8} {f.title}{mark}")
+        if f.evidence and f.severity != "info":
+            print(f"             {f.evidence[:150]}")
+
+
+async def cmd_scan(args) -> int:
+    try:
+        config = load_config(args.config)
+    except Exception as e:
+        print(f"error: config must be JSON, a JSON file path, or a URL ({e})", file=sys.stderr)
+        return 2
+    try:
+        report, _ = await scan(config, launch=not args.no_launch)
+    except ConnectError as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 3
+    findings = report.findings
+    apply_suppressions(findings, Path(args.suppress).read_text() if args.suppress else "", args.suppress)
+    if args.baseline:
+        try:
+            diff_baseline(findings, json.loads(Path(args.baseline).read_text()))
+        except Exception as e:
+            print(f"warning: baseline ignored ({e})", file=sys.stderr)
+
+    if args.format == "sarif":
+        out = json.dumps(to_sarif(findings, report.target), indent=2)
+    elif args.format == "json":
+        out = json.dumps(to_json_report(report.target, report.score, report.grade, findings,
+                                        report.manifest_hash), indent=2)
+    else:
+        console_print(report)
+        out = None
+    if out and args.output:
+        Path(args.output).write_text(out)
+        print(f"wrote {args.output}")
+    elif out:
+        print(out)
+    return gate(findings, args.fail_on)
+
+
+async def cmd_watch(args) -> int:
+    store = Store()
+    if args.run:
+        print("watching all servers; Ctrl+C to stop")
+        await watcher.run_forever(store)
+        return 0
+    if not args.name:
+        print("error: --name is required", file=sys.stderr)
+        return 2
+    if args.accept:
+        try:
+            ok = await watcher.accept(store, args.name)
+        except ConnectError as e:
+            print(f"error: {e}", file=sys.stderr)
+            return 3
+        print(f"baseline for '{args.name}' updated" if ok else f"no watch named '{args.name}'")
+        return 0 if ok else 2
+    if args.config:
+        try:
+            config = load_config(args.config)
+        except Exception as e:
+            print(f"error: {e}", file=sys.stderr)
+            return 2
+        store.add_watch(args.name, config, transport_of(config), args.interval, args.webhook)
+    watch = store.get_watch(args.name)
+    if not watch:
+        print(f"error: no watch named '{args.name}'; pass a config to create it", file=sys.stderr)
+        return 2
+    had_baseline = bool(watch.get("baseline_hash"))
+    findings = await watcher.check_watch(store, watch)
+    if not had_baseline and store.get_watch(args.name).get("baseline_hash"):
+        print(f"baseline set for '{args.name}'")
+    elif not findings:
+        print(f"'{args.name}': no changes since baseline")
+    for f in findings:
+        print(f"  [{f.severity:8}] [{f.kind or '-':8}] {f.title}")
+        if f.evidence:
+            print(f"             {f.evidence[:150]}")
+    return gate(findings, "high")
+
+
+async def cmd_crawl(args) -> int:
+    from . import crawler
+    from .crawldb import CrawlDB
+    db = CrawlDB(args.db)
+
+    def progress(done, total, what, status):
+        if status != "ok" or done % 50 == 0 or done == total:
+            print(f"  [{done}/{total}] {status:8} {what[:90]}", file=sys.stderr)
+
+    if args.action in ("sync", "run"):
+        print(f"sync: {await crawler.sync_registry(db)}", file=sys.stderr)
+    if args.action in ("probe", "run"):
+        print(f"probe: {await crawler.probe_remotes(db, concurrency=args.concurrency, min_age_h=args.min_age, limit=args.limit, per_host_cap=args.per_host_cap, progress=progress)}",
+              file=sys.stderr)
+    if args.action in ("packages", "run"):
+        print(f"packages: {await crawler.crawl_packages(db, concurrency=args.concurrency, min_age_h=args.min_age, limit=args.limit, progress=progress)}",
+              file=sys.stderr)
+    if args.action in ("advisories", "run"):
+        print(f"advisories: {await crawler.crawl_advisories(db)}", file=sys.stderr)
+    if args.action == "rescore":
+        print(f"rescored {crawler.rescore(db)} snapshots with the current rules", file=sys.stderr)
+    if args.action in ("report", "run", "rescore"):
+        md = crawler.report(db, days=args.days)
+        if args.output:
+            Path(args.output).write_text(md)
+            print(f"wrote {args.output}", file=sys.stderr)
+        else:
+            print(md)
+    return 0
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="sniffmcp")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    s = sub.add_parser("scan", help="scan one MCP server")
+    s.add_argument("config", help="inline JSON, path to a JSON file, or an MCP URL")
+    s.add_argument("--format", choices=["console", "json", "sarif"], default="console")
+    s.add_argument("--output", default=None)
+    s.add_argument("--fail-on", default="high", choices=list(SEVERITY_RANK.keys()))
+    s.add_argument("--baseline", default=None, help="previous JSON report; matched findings don't gate")
+    s.add_argument("--suppress", default=None, help="suppressions file (directives or JSON rules)")
+    s.add_argument("--offline", action="store_true", help="skip OSV malware/vulnerability lookups")
+    s.add_argument("--no-launch", action="store_true", help="never start a stdio server: config + OSV checks only")
+
+    wch = sub.add_parser("watch", help="baseline a server and report post-install changes")
+    wch.add_argument("config", nargs="?", help="config to create/update the watch")
+    wch.add_argument("--name")
+    wch.add_argument("--interval", type=int, default=3600)
+    wch.add_argument("--webhook", default=None)
+    wch.add_argument("--accept", action="store_true", help="accept the current manifest as the baseline")
+    wch.add_argument("--run", action="store_true", help="check every watch on its interval, forever")
+
+    fl = sub.add_parser("fleet", help="scan every installed MCP server from client configs")
+    fl.add_argument("paths", nargs="*", help="config file paths; omit to auto-discover")
+    fl.add_argument("--format", choices=["console", "json", "sarif"], default="console")
+    fl.add_argument("--output", default=None)
+    fl.add_argument("--fail-on", default="high", choices=list(SEVERITY_RANK.keys()))
+    fl.add_argument("--offline", action="store_true", help="skip OSV malware/vulnerability lookups")
+    fl.add_argument("--no-launch", action="store_true",
+                    help="never start stdio servers: config + OSV checks only (use in CI on untrusted configs)")
+
+    cr = sub.add_parser("crawl", help="crawl the official MCP registry and snapshot public servers over time")
+    cr.add_argument("action", choices=["sync", "probe", "packages", "advisories", "report", "run", "rescore"],
+                    help="run = sync + probe + packages + advisories + report (what a daily cron should call)")
+    cr.add_argument("--db", default=None, help="crawl database (default ~/.sniffmcp/crawl.db)")
+    cr.add_argument("--limit", type=int, default=None, help="probe/check at most N items")
+    cr.add_argument("--concurrency", type=int, default=8)
+    cr.add_argument("--min-age", type=float, default=20, help="hours before an item is re-probed")
+    cr.add_argument("--per-host-cap", type=int, default=100, help="max endpoints probed per host per run (rotates)")
+    cr.add_argument("--days", type=int, default=7, help="report window")
+    cr.add_argument("--output", default=None, help="write the report here instead of stdout")
+
+    sv = sub.add_parser("serve", help="run as an MCP server")
+    sv.add_argument("--http", action="store_true")
+    sv.add_argument("--port", type=int, default=8930)
+
+    args = ap.parse_args(argv)
+    if getattr(args, "offline", False):
+        os.environ["SNIFFMCP_OFFLINE"] = "1"
+    if args.cmd == "fleet":
+        try:
+            fleet = asyncio.run(scan_fleet(args.paths or None, args.fail_on, launch=not args.no_launch))
+        except ValueError as e:
+            print(f"error: {e}", file=sys.stderr)
+            return 2
+        if args.format in ("json", "sarif"):
+            out = json.dumps(fleet if args.format == "json" else fleet_to_sarif(fleet), indent=2)
+            if args.output:
+                Path(args.output).write_text(out)
+                print(f"wrote {args.output}")
+            else:
+                print(out)
+        else:
+            fleet_console(fleet)
+        return fleet["summary"]["gate"]
+    if args.cmd == "scan":
+        return asyncio.run(cmd_scan(args))
+    if args.cmd == "watch":
+        return asyncio.run(cmd_watch(args))
+    if args.cmd == "crawl":
+        return asyncio.run(cmd_crawl(args))
+    if args.cmd == "serve":
+        from .server import mcp   # mcp.run() owns its event loop; don't wrap it in asyncio.run
+        if args.http:
+            mcp.run(transport="streamable-http", host="127.0.0.1", port=args.port)
+        else:
+            mcp.run(transport="stdio")
+        return 0
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

sniffmcp/client.py

@@ -0,0 +1,160 @@
+"""Connect to a target MCP server and pull the manifest the agent would see.
+
+Uses the official SDK (mcp>=2). `Client(mode="auto")` probes the 2026-07-28
+`server/discover` and falls back to the legacy initialize handshake, so both
+protocol generations scan the same way.
+
+Config shapes accepted (same as Claude / Cursor / Claude Code configs):
+  {"command": "npx", "args": ["-y", "pkg@1.2.3"], "env": {...}, "cwd": "..."}
+  {"url": "https://host/mcp", "headers": {...}}                 # Streamable HTTP
+  {"type": "sse", "url": "https://host/sse", "headers": {...}}  # legacy SSE
+Optional: "timeout" (seconds).
+"""
+from __future__ import annotations
+import os, re, tempfile
+
+import anyio
+from mcp import Client
+from mcp.client.stdio import StdioServerParameters, stdio_client
+from mcp.client.streamable_http import streamable_http_client
+from mcp.client.sse import sse_client
+from mcp.shared._httpx_utils import create_mcp_http_client
+from mcp.types import Implementation
+
+from . import __version__
+
+DEFAULT_TIMEOUT_STDIO = 90.0   # first `npx -y` / `uvx` run downloads the package
+DEFAULT_TIMEOUT_HTTP = 30.0
+MAX_PAGES = 50
+
+
+class ConnectError(Exception):
+    """Target could not be reached, spawned, or did not complete the handshake."""
+
+
+_VAR = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")
+
+
+def _expand(value: str) -> str:
+    """Expand ${VAR} and ${VAR:-default} the way Claude Code / Cursor configs do."""
+    return _VAR.sub(lambda m: os.environ.get(m.group(1), m.group(2) or ""), value)
+
+
+def transport_of(config: dict) -> str:
+    if "url" not in config:
+        return "stdio"
+    if config.get("type") == "sse" or config.get("transport") == "sse":
+        return "sse"
+    return "streamable_http"
+
+
+def _innermost(e: BaseException) -> BaseException:
+    while isinstance(e, BaseExceptionGroup) and e.exceptions:
+        e = e.exceptions[0]
+    return e
+
+
+async def _list_all(fetch, attr: str) -> list[dict]:
+    out, cursor = [], None
+    for _ in range(MAX_PAGES):
+        page = await fetch(cursor=cursor)
+        out.extend(x.model_dump(mode="json", by_alias=True, exclude_none=True)
+                   for x in getattr(page, attr))
+        cursor = page.next_cursor
+        if not cursor:
+            break
+    return out
+
+
+async def _collect(client: Client) -> dict:
+    caps = client.server_capabilities
+    manifest = {"tools": [], "resources": [], "prompts": [],
+                "server_info": {}, "instructions": client.instructions or "",
+                "protocol_version": client.protocol_version, "errors": []}
+    if client.server_info:
+        manifest["server_info"] = client.server_info.model_dump(mode="json", exclude_none=True)
+    # Only ask for what the server advertises; some servers error on unknown methods.
+    for attr, cap, fetch in (("tools", caps.tools, client.list_tools),
+                             ("resources", caps.resources, client.list_resources),
+                             ("prompts", caps.prompts, client.list_prompts)):
+        if cap is None:
+            continue
+        try:
+            manifest[attr] = await _list_all(fetch, attr)
+        except Exception as e:  # a failed list is reported, not fatal
+            manifest["errors"].append(f"{attr}/list failed: {_innermost(e)}")
+    return manifest
+
+
+def _transport_for(config: dict, errlog):
+    kind = transport_of(config)
+    if kind == "stdio":
+        cmd = config.get("command")
+        if not cmd or not isinstance(cmd, str):
+            raise ConnectError("config has neither 'command' nor 'url'")
+        env = {k: _expand(str(v)) for k, v in (config.get("env") or {}).items()}
+        params = StdioServerParameters(command=_expand(cmd),
+                                       args=[_expand(str(a)) for a in config.get("args") or []],
+                                       env=env or None, cwd=config.get("cwd"))
+        return stdio_client(params, errlog=errlog)
+    url = _expand(config["url"])
+    headers = {k: _expand(str(v)) for k, v in (config.get("headers") or {}).items()}
+    if kind == "sse":
+        return sse_client(url, headers=headers or None)
+    return streamable_http_client(url, http_client=create_mcp_http_client(headers=headers or None))
+
+
+async def fetch_manifest(config: dict, client_name: str = "sniffmcp") -> dict:
+    """Connect, negotiate, and return tools/resources/prompts/server_info/instructions."""
+    kind = transport_of(config)
+    timeout = float(config.get("timeout") or
+                    (DEFAULT_TIMEOUT_STDIO if kind == "stdio" else DEFAULT_TIMEOUT_HTTP))
+    with tempfile.TemporaryFile(mode="w+", encoding="utf-8", errors="replace") as errlog:
+        try:
+            transport = _transport_for(config, errlog)
+            with anyio.fail_after(timeout):
+                async with Client(transport, mode="auto", cache=None,
+                                  client_info=Implementation(name=client_name, version=__version__)) as client:
+                    return await _collect(client)
+        except ConnectError:
+            raise
+        except TimeoutError:
+            raise ConnectError(f"timed out after {timeout:.0f}s{_stderr_tail(errlog)}") from None
+        except Exception as e:
+            inner = _innermost(e)
+            if kind != "stdio":
+                hint = await _http_status_hint(config)
+                if hint:
+                    raise ConnectError(hint) from None
+            raise ConnectError(f"{type(inner).__name__}: {inner}{_stderr_tail(errlog)}") from None
+
+
+async def _http_status_hint(config: dict) -> str | None:
+    """After a failed remote connect, say *why* in terms a user can act on."""
+    import httpx2
+    headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream",
+               **{k: _expand(str(v)) for k, v in (config.get("headers") or {}).items()}}
+    body = {"jsonrpc": "2.0", "id": 1, "method": "initialize",
+            "params": {"protocolVersion": "2025-06-18", "capabilities": {},
+                       "clientInfo": {"name": "sniffmcp", "version": "0"}}}
+    try:
+        async with httpx2.AsyncClient(timeout=10) as http:
+            r = await http.post(_expand(config["url"]), json=body, headers=headers)
+    except Exception:
+        return None
+    if r.status_code in (401, 403):
+        oauth = "resource_metadata" in r.headers.get("www-authenticate", "")
+        return (f"requires authentication (HTTP {r.status_code}"
+                f"{', OAuth' if oauth else ''}); pass credentials in \"headers\" to scan it")
+    if r.status_code >= 400:
+        return f"HTTP {r.status_code} from server"
+    return None
+
+
+def _stderr_tail(errlog) -> str:
+    try:
+        errlog.seek(0)
+        lines = [l.strip() for l in errlog.read().splitlines() if l.strip()]
+    except Exception:
+        return ""
+    return (" | stderr: " + " / ".join(lines[-3:])[:300]) if lines else ""

sniffmcp/crawldb.py

@@ -0,0 +1,121 @@
+"""SQLite store for the ecosystem crawl: registry entries, remote endpoint
+snapshots, manifest changes, and npm/PyPI release history.
+
+Snapshots are stored only when a manifest changes; every probe is logged
+separately so availability and auth stats stay honest.
+Default path ~/.sniffmcp/crawl.db (override with SNIFFMCP_CRAWL_DB or --db).
+"""
+from __future__ import annotations
+import json, os, sqlite3, time
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS servers(
+  name TEXT PRIMARY KEY, version TEXT, title TEXT, description TEXT, repo_url TEXT,
+  status TEXT, published_at TEXT, updated_at TEXT, raw TEXT, first_seen REAL, last_seen REAL);
+CREATE TABLE IF NOT EXISTS server_versions(
+  name TEXT, version TEXT, published_at TEXT, first_seen REAL, PRIMARY KEY(name, version));
+CREATE TABLE IF NOT EXISTS endpoints(
+  id INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT UNIQUE, transport TEXT, server_name TEXT,
+  auth_declared INTEGER, skip_reason TEXT, first_seen REAL, last_seen REAL);
+CREATE TABLE IF NOT EXISTS probes(
+  id INTEGER PRIMARY KEY AUTOINCREMENT, endpoint_id INTEGER, taken_at REAL, status TEXT,
+  error TEXT, manifest_hash TEXT, duration_ms INTEGER);
+CREATE TABLE IF NOT EXISTS snapshots(
+  id INTEGER PRIMARY KEY AUTOINCREMENT, endpoint_id INTEGER, taken_at REAL, manifest_hash TEXT,
+  manifest TEXT, protocol_version TEXT, tool_count INTEGER, score INTEGER, grade TEXT, findings TEXT);
+CREATE TABLE IF NOT EXISTS changes(
+  id INTEGER PRIMARY KEY AUTOINCREMENT, endpoint_id INTEGER, detected_at REAL, from_hash TEXT,
+  to_hash TEXT, kind TEXT, severity TEXT, title TEXT, tool_name TEXT, evidence TEXT);
+CREATE TABLE IF NOT EXISTS packages(
+  ecosystem TEXT, name TEXT, server_name TEXT, first_seen REAL, last_checked REAL, error TEXT,
+  PRIMARY KEY(ecosystem, name));
+CREATE TABLE IF NOT EXISTS package_versions(
+  ecosystem TEXT, name TEXT, version TEXT, published_at TEXT, install_scripts TEXT,
+  publisher TEXT, provenance INTEGER, sdist_only INTEGER, integrity TEXT, first_seen REAL,
+  PRIMARY KEY(ecosystem, name, version));
+CREATE TABLE IF NOT EXISTS package_events(
+  id INTEGER PRIMARY KEY AUTOINCREMENT, ecosystem TEXT, name TEXT, version TEXT,
+  published_at TEXT, detected_at REAL, kind TEXT, severity TEXT, detail TEXT,
+  UNIQUE(ecosystem, name, version, kind));
+CREATE TABLE IF NOT EXISTS package_advisories(
+  ecosystem TEXT, name TEXT, vuln_id TEXT, kind TEXT, severity TEXT, summary TEXT, aliases TEXT,
+  affects_latest INTEGER, latest_version TEXT, first_seen REAL, last_seen REAL,
+  PRIMARY KEY(ecosystem, name, vuln_id));
+CREATE INDEX IF NOT EXISTS probes_ep ON probes(endpoint_id, taken_at);
+CREATE INDEX IF NOT EXISTS snaps_ep ON snapshots(endpoint_id, id);
+"""
+
+
+def default_path() -> str:
+    return os.environ.get("SNIFFMCP_CRAWL_DB") or os.path.join(
+        os.path.expanduser("~"), ".sniffmcp", "crawl.db")
+
+
+class CrawlDB:
+    def __init__(self, path: str | None = None):
+        path = path or default_path()
+        if path != ":memory:":
+            os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
+        self.conn = sqlite3.connect(path, timeout=60)   # wait for other writers (e.g. a concurrent rescore) instead of failing
+        self.conn.row_factory = sqlite3.Row
+        self.conn.execute("PRAGMA journal_mode=WAL")
+        self.conn.executescript(SCHEMA)
+        self.conn.commit()
+
+    def q(self, sql, *args) -> list[dict]:
+        return [dict(r) for r in self.conn.execute(sql, args)]
+
+    def one(self, sql, *args):
+        row = self.conn.execute(sql, args).fetchone()
+        return row[0] if row else None
+
+    # ---- registry ----
+    def upsert_server(self, entry: dict, now: float) -> None:
+        s, meta = entry["server"], entry.get("_meta", {}).get("io.modelcontextprotocol.registry/official", {})
+        self.conn.execute(
+            "INSERT INTO servers(name,version,title,description,repo_url,status,published_at,updated_at,raw,first_seen,last_seen)"
+            " VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(name) DO UPDATE SET version=excluded.version,"
+            " title=excluded.title, description=excluded.description, repo_url=excluded.repo_url,"
+            " status=excluded.status, published_at=excluded.published_at, updated_at=excluded.updated_at,"
+            " raw=excluded.raw, last_seen=excluded.last_seen",
+            (s["name"], s.get("version"), s.get("title"), s.get("description"),
+             (s.get("repository") or {}).get("url"), meta.get("status"), meta.get("publishedAt"),
+             meta.get("updatedAt"), json.dumps(s), now, now))
+        self.conn.execute("INSERT OR IGNORE INTO server_versions(name,version,published_at,first_seen) VALUES(?,?,?,?)",
+                          (s["name"], s.get("version"), meta.get("publishedAt"), now))
+
+    def upsert_endpoint(self, url, transport, server_name, auth_declared, skip_reason, now) -> None:
+        self.conn.execute(
+            "INSERT INTO endpoints(url,transport,server_name,auth_declared,skip_reason,first_seen,last_seen)"
+            " VALUES(?,?,?,?,?,?,?) ON CONFLICT(url) DO UPDATE SET transport=excluded.transport,"
+            " server_name=excluded.server_name, auth_declared=excluded.auth_declared,"
+            " skip_reason=excluded.skip_reason, last_seen=excluded.last_seen",
+            (url, transport, server_name, int(auth_declared), skip_reason, now, now))
+
+    def upsert_package(self, ecosystem, name, server_name, now) -> None:
+        self.conn.execute("INSERT OR IGNORE INTO packages(ecosystem,name,server_name,first_seen) VALUES(?,?,?,?)",
+                          (ecosystem, name, server_name, now))
+
+    # ---- probes / snapshots ----
+    def due_endpoints(self, min_age_s: float, limit: int | None, per_host_cap: int | None = None) -> list[dict]:
+        """Stalest first (never-probed, then oldest), at most per_host_cap per host per run.
+        Bulk publishers register thousands of entries on one host; the cap samples them
+        on rotation instead of hitting one operator thousands of times a day."""
+        from urllib.parse import urlparse
+        sql = ("SELECT e.*, COALESCE((SELECT MAX(taken_at) FROM probes p WHERE p.endpoint_id=e.id), 0) AS last_probe"
+               " FROM endpoints e WHERE e.skip_reason IS NULL AND last_probe < ? ORDER BY last_probe, e.id")
+        rows, per_host = [], {}
+        for r in self.q(sql, time.time() - min_age_s):
+            host = urlparse(r["url"]).hostname or ""
+            if per_host_cap and per_host.get(host, 0) >= per_host_cap:
+                continue
+            per_host[host] = per_host.get(host, 0) + 1
+            rows.append(r)
+        return rows[:limit] if limit else rows
+
+    def last_snapshot(self, endpoint_id) -> dict | None:
+        rows = self.q("SELECT * FROM snapshots WHERE endpoint_id=? ORDER BY id DESC LIMIT 1", endpoint_id)
+        return rows[0] if rows else None
+
+    def commit(self):
+        self.conn.commit()