scrapeunblocker-mcp 0.1.4 → 0.2.1

package/README.md

@@ -11,10 +11,51 @@ You bring your **own** API key. Nothing is shared or proxied through us.
 
 | Tool | What it does |
 |------|--------------|
-| `fetch_html` | Fetch the fully rendered HTML of a URL. |
+| `fetch_html` | Fetch the fully rendered HTML of a URL (optionally after running interactive browser `steps`). |
+| `list_elements` | List a page's notable elements with a ready-to-use selector for each (selector discovery). |
 | `fetch_parsed` | Fetch a page and return AI-parsed structured JSON. |
 | `google_search` | Run a Google search and return organic results as JSON. |
 
+### Browser steps (interact, then capture)
+
+Some pages only reveal what you need after you interact with them - accept a
+cookie banner, click a tab, type into a search box and submit, or scroll to
+trigger lazy loading. Pass an optional `steps` array to `fetch_html` and those
+actions run in a **real browser**, in order, after the page loads; the resulting
+HTML is then returned.
+
+Available actions:
+
+| Action | Fields |
+|--------|--------|
+| `wait_for` | `selector`, `selector_type?` (`css`/`xPath`/`className`/`tagName`), `timeout_ms?` |
+| `wait_for_text` | `value`, `timeout_ms?` |
+| `wait` | `value` (ms) |
+| `click` | `selector`, `selector_type?`, `timeout_ms?` |
+| `type` | `selector`, `selector_type?`, `value`, `clear?`, `timeout_ms?` (typed human-like) |
+| `select` | `selector`, `selector_type?`, `value`, `timeout_ms?` |
+| `press_key` | `value` (`Enter`, `Tab`, `Escape`, `Backspace`, `Delete`, `Space`, `Arrow*`, `Home`, `End`, `PageUp`, `PageDown`) |
+| `scroll` | `value` (`"bottom"` or a pixel offset) |
+
+Steps are **not idempotent** - they run once per call. If a step fails,
+`fetch_html` returns which step failed, why, and the page HTML at that moment so
+you can fix the selector and retry.
+
+### List elements (discover selectors first)
+
+`list_elements` loads a page and returns a JSON list of its notable elements
+(links, inputs, buttons, selects, ...), each with a ready-to-use `selector` plus
+`tag`, `text` and useful attributes (`name`, `id`, `type`, `placeholder`,
+`aria_label`, `href`, ...):
+
+```json
+{ "url": "https://example.com", "count": 42, "elements": [ { "tag": "input", "selector": "#search", "type": "text", "placeholder": "Search", "aria_label": "Search" } ] }
+```
+
+The natural workflow is **discover, then act**: call `list_elements` to find the
+selectors you need, then pass matching `steps` to `fetch_html` to click/type/
+select and capture the resulting HTML.
+
 ## Get an API key
 
 Sign up and grab your key at **[app.scrapeunblocker.com](https://app.scrapeunblocker.com?utm_source=mcp&utm_medium=integration&utm_campaign=mcp-server)**. The server
@@ -76,6 +117,7 @@ SCRAPEUNBLOCKER_KEY=your_api_key_here npx -y scrapeunblocker-mcp
 
 - "Fetch the HTML of https://www.example-shop.com/product/123 and list the price."
 - "This page keeps blocking me: <url>. Use fetch_html to get it."
+- "List the elements on <url>, then use fetch_html steps to type 'laptop' into the search box, press Enter, wait for the results, and give me the HTML."
 - "Search Google for 'best running shoes 2026' and give me the top 5 links."
 
 ## Development

package/dist/index.js

@@ -4,12 +4,20 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
-import { ScrapeUnblockerClient } from "scrapeunblocker";
-var VERSION = "0.1.2";
+import { APIError, ScrapeUnblockerClient } from "scrapeunblocker";
+var VERSION = "0.2.1";
+var DEFAULT_BASE_URL = "https://api.scrapeunblocker.com";
+var API_KEY_HEADER = "x-scrapeunblocker-key";
 function resolveApiKey() {
   return process.env.SCRAPEUNBLOCKER_KEY ?? process.env.SCRAPEUNBLOCKER_API_KEY ?? void 0;
 }
 var MISSING_KEY_MESSAGE = "No ScrapeUnblocker API key found. Set the SCRAPEUNBLOCKER_KEY environment variable to your own key (get one at https://app.scrapeunblocker.com) and restart the MCP server.";
+function resolveBaseUrl() {
+  return (process.env.SCRAPEUNBLOCKER_BASE_URL || DEFAULT_BASE_URL).replace(
+    /\/+$/,
+    ""
+  );
+}
 function client() {
   const apiKey = resolveApiKey();
   if (!apiKey) throw new Error(MISSING_KEY_MESSAGE);
@@ -20,15 +28,128 @@ function errorText(err) {
   if (err instanceof Error) return err.message;
   return String(err);
 }
+var TARGET_GONE = /* @__PURE__ */ new Set([404, 410]);
+function targetGoneStatus(status, originStatus) {
+  const origin = Number(originStatus);
+  if (TARGET_GONE.has(origin)) return origin;
+  return TARGET_GONE.has(status) ? status : null;
+}
+function targetGoneText(url, code, page) {
+  const reason = code === 410 ? "Gone" : "Not Found";
+  const head = `The target page does not exist: ${url} answered HTTP ${code} (${reason}). This is the website's own answer, not a block or an API failure. The call was billed, and retrying returns the same result - check the URL instead.`;
+  return page.trim() ? `${head}
+
+The target's own ${code} page follows:
+
+${page}` : `${head}
+
+The target sent no page body.`;
+}
+function errorResult(url, err) {
+  const gone = err instanceof APIError ? targetGoneStatus(err.statusCode) : null;
+  if (gone !== null) {
+    const body = err instanceof APIError ? err.body ?? "" : "";
+    return { content: [{ type: "text", text: targetGoneText(url, gone, body) }] };
+  }
+  return { content: [{ type: "text", text: errorText(err) }], isError: true };
+}
+async function getPageSourceRaw(params) {
+  const apiKey = resolveApiKey();
+  if (!apiKey) throw new Error(MISSING_KEY_MESSAGE);
+  const query = new URLSearchParams();
+  for (const [key, value] of Object.entries(params)) {
+    if (value === void 0 || value === null) continue;
+    query.set(key, String(value));
+  }
+  const url = `${resolveBaseUrl()}/getPageSource?${query.toString()}`;
+  const response = await fetch(url, {
+    method: "POST",
+    headers: {
+      [API_KEY_HEADER]: apiKey,
+      "User-Agent": `scrapeunblocker-mcp/${VERSION}`,
+      Accept: "*/*"
+    }
+  });
+  const text = await response.text();
+  return {
+    status: response.status,
+    ok: response.ok,
+    text,
+    originStatus: response.headers.get("x-origin-status")
+  };
+}
 var server = new McpServer({
   name: "scrapeunblocker",
   version: VERSION
 });
+var selectorTypeSchema = z.enum(["css", "xPath", "className", "tagName"]).describe("How `selector` is interpreted. Defaults to 'css'.");
+var pressableKeySchema = z.enum([
+  "Enter",
+  "Tab",
+  "Escape",
+  "Backspace",
+  "Delete",
+  "Space",
+  "ArrowUp",
+  "ArrowDown",
+  "ArrowLeft",
+  "ArrowRight",
+  "Home",
+  "End",
+  "PageUp",
+  "PageDown"
+]);
+var stepSchema = z.discriminatedUnion("action", [
+  z.object({
+    action: z.literal("wait_for"),
+    selector: z.string().describe("Selector to wait for (until present)."),
+    selector_type: selectorTypeSchema.optional(),
+    timeout_ms: z.number().int().positive().optional()
+  }).describe("Wait until an element matching `selector` exists."),
+  z.object({
+    action: z.literal("wait_for_text"),
+    value: z.string().describe("Text to wait for anywhere on the page."),
+    timeout_ms: z.number().int().positive().optional()
+  }).describe("Wait until the given text appears on the page."),
+  z.object({
+    action: z.literal("wait"),
+    value: z.number().int().nonnegative().describe("Fixed pause in milliseconds.")
+  }).describe("Pause for a fixed number of milliseconds."),
+  z.object({
+    action: z.literal("click"),
+    selector: z.string().describe("Selector of the element to click."),
+    selector_type: selectorTypeSchema.optional(),
+    timeout_ms: z.number().int().positive().optional()
+  }).describe("Click the element matching `selector`."),
+  z.object({
+    action: z.literal("type"),
+    selector: z.string().describe("Selector of the field to type into."),
+    selector_type: selectorTypeSchema.optional(),
+    value: z.string().describe("Text to type (entered human-like)."),
+    clear: z.boolean().optional().describe("Clear the field before typing."),
+    timeout_ms: z.number().int().positive().optional()
+  }).describe("Type text into an input, character by character."),
+  z.object({
+    action: z.literal("select"),
+    selector: z.string().describe("Selector of the <select> element."),
+    selector_type: selectorTypeSchema.optional(),
+    value: z.string().describe("The option value to select."),
+    timeout_ms: z.number().int().positive().optional()
+  }).describe("Choose an option in a <select> dropdown by value."),
+  z.object({
+    action: z.literal("press_key"),
+    value: pressableKeySchema.describe("The key to press.")
+  }).describe("Press a single keyboard key."),
+  z.object({
+    action: z.literal("scroll"),
+    value: z.union([z.literal("bottom"), z.number().int()]).describe("'bottom' to scroll to the end, or a pixel offset.")
+  }).describe("Scroll the page to the bottom or by a pixel amount.")
+]);
 server.registerTool(
   "fetch_html",
   {
     title: "Fetch page HTML",
-    description: "Fetch the fully rendered HTML of any web page through ScrapeUnblocker, bypassing anti-bot protection (Cloudflare, DataDome, PerimeterX, Akamai, Shape). Use this when a normal fetch is blocked (403/429, captcha, 'access denied') or when the page needs a real browser to render. Returns the raw HTML as text.",
+    description: "Fetch the fully rendered HTML of any web page through ScrapeUnblocker, bypassing anti-bot protection (Cloudflare, DataDome, PerimeterX, Akamai, Shape). Use this when a normal fetch is blocked (403/429, captcha, 'access denied') or when the page needs a real browser to render. Returns the raw HTML as text.\n\nFor pages that need interaction (accept a cookie banner, click a tab, type into a search box, scroll to trigger lazy loading) pass `steps`: an ordered list of browser actions run in a real browser AFTER the page loads, then the resulting HTML is returned. Workflow: first call the `list_elements` tool to discover the real selectors on the page, then build `steps` against them. Steps are NOT idempotent - they run once per call. If a step fails, this tool returns which step failed, why, and the page HTML at that moment so you can fix the selector and retry.",
     inputSchema: {
       url: z.string().url().describe("The absolute URL to fetch (http/https)."),
       proxy_country: z.string().length(2).optional().describe(
@@ -41,21 +162,114 @@ server.registerTool(
         `The CSS selector or JS expression paired with wait_method (e.g. '#price' or 'document.readyState==="complete"').`
       ),
       method_timeout_seconds: z.number().int().positive().optional().describe("Cap in seconds for the render-wait method."),
-      sleep_seconds: z.number().positive().optional().describe("Extra seconds to wait after load before capturing the HTML.")
+      sleep_seconds: z.number().positive().optional().describe("Extra seconds to wait after load before capturing the HTML."),
+      steps: z.array(stepSchema).optional().describe(
+        "Optional ordered browser actions to run in a real browser after the page loads, before the HTML is captured. Use `list_elements` first to find selectors. Runs once (not idempotent)."
+      )
     }
   },
   async (args) => {
     try {
-      const html = await client().getPageSource(args.url, {
-        proxyCountry: args.proxy_country,
+      if (!args.steps || args.steps.length === 0) {
+        const html = await client().getPageSource(args.url, {
+          proxyCountry: args.proxy_country,
+          method: args.wait_method,
+          value: args.wait_value,
+          methodTimeout: args.method_timeout_seconds,
+          timeSleep: args.sleep_seconds
+        });
+        return { content: [{ type: "text", text: html }] };
+      }
+      const res = await getPageSourceRaw({
+        url: args.url,
+        proxy_country: args.proxy_country,
         method: args.wait_method,
         value: args.wait_value,
-        methodTimeout: args.method_timeout_seconds,
-        timeSleep: args.sleep_seconds
+        method_timeout: args.method_timeout_seconds,
+        time_sleep: args.sleep_seconds,
+        steps: JSON.stringify(args.steps)
       });
-      return { content: [{ type: "text", text: html }] };
+      const gone = targetGoneStatus(res.status, res.originStatus);
+      if (gone !== null) {
+        return { content: [{ type: "text", text: targetGoneText(args.url, gone, res.text) }] };
+      }
+      if (res.ok) {
+        return { content: [{ type: "text", text: res.text }] };
+      }
+      if (res.status === 422) {
+        let detail = {};
+        try {
+          detail = JSON.parse(res.text);
+        } catch {
+        }
+        if (detail.error === "step_failed") {
+          const idx = typeof detail.step_index === "number" ? detail.step_index : void 0;
+          const summary = `Browser step failed` + (idx !== void 0 ? ` at index ${idx}` : "") + (detail.action ? ` (${detail.action})` : "") + (detail.selector ? ` on selector ${JSON.stringify(detail.selector)}` : "") + `: ${detail.reason ?? "unknown reason"}.
+
+Fix the selector or step (use list_elements to inspect the page) and retry. The page HTML at the point of failure follows:
+
+` + (detail.html ?? "");
+          return { content: [{ type: "text", text: summary }], isError: true };
+        }
+        return {
+          content: [{ type: "text", text: res.text }],
+          isError: true
+        };
+      }
+      return {
+        content: [
+          {
+            type: "text",
+            text: `ScrapeUnblocker returned HTTP ${res.status}: ${res.text.slice(0, 500)}`
+          }
+        ],
+        isError: true
+      };
     } catch (err) {
-      return { content: [{ type: "text", text: errorText(err) }], isError: true };
+      return errorResult(args.url, err);
+    }
+  }
+);
+server.registerTool(
+  "list_elements",
+  {
+    title: "List page elements (selector discovery)",
+    description: "Load a page through ScrapeUnblocker and return a JSON list of its notable elements (links, inputs, buttons, selects, etc.) with a ready-to-use `selector` for each, plus tag, text and useful attributes (name, id, type, placeholder, aria_label, href, ...). Read-only and does not interact with the page. This is the discovery half of interactive scraping: call `list_elements` to find the selectors you need, then pass matching `steps` to `fetch_html` to click/type/select and capture the resulting HTML.",
+    inputSchema: {
+      url: z.string().url().describe("The absolute URL whose elements you want to list."),
+      proxy_country: z.string().length(2).optional().describe("Optional ISO country code to route through, e.g. 'US'.")
+    }
+  },
+  async (args) => {
+    try {
+      const res = await getPageSourceRaw({
+        url: args.url,
+        list_elements: true,
+        proxy_country: args.proxy_country
+      });
+      const gone = targetGoneStatus(res.status, res.originStatus);
+      if (!res.ok && gone === null) {
+        return {
+          content: [
+            {
+              type: "text",
+              text: `ScrapeUnblocker returned HTTP ${res.status}: ${res.text.slice(0, 500)}`
+            }
+          ],
+          isError: true
+        };
+      }
+      let out = res.text;
+      try {
+        out = JSON.stringify(JSON.parse(res.text), null, 2);
+      } catch {
+      }
+      if (gone !== null) {
+        return { content: [{ type: "text", text: targetGoneText(args.url, gone, out) }] };
+      }
+      return { content: [{ type: "text", text: out }] };
+    } catch (err) {
+      return errorResult(args.url, err);
     }
   }
 );
@@ -82,7 +296,7 @@ server.registerTool(
         content: [{ type: "text", text: JSON.stringify(parsed, null, 2) }]
       };
     } catch (err) {
-      return { content: [{ type: "text", text: errorText(err) }], isError: true };
+      return errorResult(args.url, err);
     }
   }
 );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrapeunblocker-mcp",
-  "version": "0.1.4",
+  "version": "0.2.1",
   "mcpName": "com.scrapeunblocker/scrapeunblocker-mcp",
   "description": "Model Context Protocol (MCP) server for ScrapeUnblocker - lets Claude fetch any web page's HTML (or AI-parsed JSON) through the ScrapeUnblocker anti-bot API, using your own API key.",
   "keywords": [