scrapeunblocker-mcp 0.3.5 → 0.5.0

package/README.md

@@ -16,8 +16,11 @@ You bring your **own** API key. Nothing is shared or proxied through us.
 | `list_elements` | List a page's notable elements with a ready-to-use selector for each (selector discovery). |
 | `fetch_parsed` | Fetch a page and return AI-parsed structured JSON. |
 | `google_search` | Run a Google search and return organic results as JSON. |
+| `fetch_markdown` | Fetch a page and return it as clean Markdown: the main content with absolute links, without scripts, styles, navigation or footer. Usually 2-25% the size of the HTML, so it is the cheap way for a model to read a page. |
+| `search_and_read` | Search Google and read the top results in one call: Google's AI Overview (when there is one), the first 1-5 result pages as Markdown, and the other results as title, URL and snippet. Billed as one search plus one request per page read. |
 | `list_plugins` | List the ready-made site plugins (100+: Amazon, Walmart, Zillow, Booking.com, Indeed, Google Trends, YouTube, App Store, Ahrefs, ...) with their parameters. Free - it calls nothing. |
 | `run_plugin` | Run one site plugin and return its structured JSON. |
+| `amazon_search`, `amazon_product`, `amazon_reviews`, `walmart_search`, `ebay_search`, `google_shopping`, `google_local`, `maps_place_details`, `google_images`, `youtube_search`, `youtube_transcript`, `zillow_search`, `indeed_jobs`, `trustpilot_reviews`, `booking_hotels` | The most-used site plugins as tools of their own, each with its parameters in the tool schema - one call, no `list_plugins` lookup. Same JSON as `run_plugin`. |
 
 ### Site plugins (structured JSON for known sites)
 
@@ -25,7 +28,8 @@ For many popular sites there is a plugin that returns the data already
 parsed - a Target search, a Zillow listing search, an app's App Store reviews,
 Google Trends for a few keywords. The model calls `list_plugins` (optionally
 with `search: "zillow"` or a `category`) to find the plugin and its
-parameters, then `run_plugin` with `name` and `params`:
+parameters, then `run_plugin` with `name` and `params` (the most-used plugins, listed in
+the table above, can also be called directly as their own tools):
 
 ```json
 { "name": "target_search", "params": { "query": "coffee maker", "max_results": 20 } }
@@ -138,6 +142,8 @@ SCRAPEUNBLOCKER_KEY=your_api_key_here npx -y scrapeunblocker-mcp
 - "This page keeps blocking me: <url>. Use fetch_html to get it."
 - "List the elements on <url>, then use fetch_html steps to type 'laptop' into the search box, press Enter, wait for the results, and give me the HTML."
 - "Search Google for 'best running shoes 2026' and give me the top 5 links."
+- "Read https://en.wikipedia.org/wiki/Web_scraping as Markdown and summarise it in five bullets."
+- "Search the web for the current population of Vilnius and cite the pages you read."
 - "Find coffee makers on Target under $50 and compare them with Walmart's."
 - "Show Google Trends for 'coffee' vs 'tea' in the US over the last year."
 - "What are the newest App Store reviews of WhatsApp saying?"

package/dist/index.js

@@ -3,7 +3,7 @@
 // src/index.ts
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
-import { z as z2 } from "zod";
+import { z as z3 } from "zod";
 import { APIError, ScrapeUnblockerClient } from "scrapeunblocker";
 
 // src/plugin-tools.ts
@@ -5506,6 +5506,23 @@ var MAX_OUTPUT_CHARS = 1e5;
 var BY_NAME = new Map(PLUGINS.map((p) => [p.name, p]));
 var NAMES = PLUGINS.map((p) => p.name);
 var CATEGORIES2 = CATEGORIES;
+var FEATURED_PLUGINS = [
+  "amazon_search",
+  "amazon_product",
+  "amazon_reviews",
+  "walmart_search",
+  "ebay_search",
+  "google_shopping",
+  "google_local",
+  "maps_place_details",
+  "google_images",
+  "youtube_search",
+  "youtube_transcript",
+  "zillow_search",
+  "indeed_jobs",
+  "trustpilot_reviews",
+  "booking_hotels"
+];
 var text = (t, isError = false) => isError ? { content: [{ type: "text", text: t }], isError: true } : { content: [{ type: "text", text: t }] };
 function signature(p) {
   return `${p.name}(${p.params.map((q) => q.required ? `${q.name}*` : q.name).join(", ")})`;
@@ -5564,8 +5581,49 @@ function prepare(p, params) {
   }
   return out;
 }
+function paramType(q) {
+  const scalar = (t) => t === "integer" ? z.number().int() : t === "number" ? z.number() : t === "boolean" ? z.boolean() : z.string();
+  let type;
+  if (q.type === "array") {
+    const item = scalar(q.items);
+    type = z.union([z.array(item), item]);
+  } else if (q.enum && q.enum.length && q.enum.every((v) => typeof v === "string")) {
+    type = z.enum(q.enum);
+    if (q.join) type = z.union([type, z.array(type)]);
+  } else {
+    type = scalar(q.type);
+    if (q.join) type = z.union([type, z.array(type)]);
+  }
+  const extras = [
+    q.enum && !q.enum.every((v) => typeof v === "string") ? `One of: ${q.enum.join(", ")}.` : "",
+    q.default !== void 0 ? `Default: ${String(q.default)}.` : ""
+  ].filter(Boolean);
+  const description = [q.description, ...extras].filter(Boolean).join(" ");
+  if (description) type = type.describe(description);
+  return q.required ? type : type.optional();
+}
 function registerPluginTools(server2, options) {
   const annotations = { readOnlyHint: true, openWorldHint: true, destructiveHint: false, ...options.annotations };
+  async function run(plugin, raw) {
+    const blocked = options.blocked?.();
+    if (blocked) return blocked;
+    const params = prepare(plugin, raw);
+    if (typeof params === "string") return text(`${params}
+
+${details(plugin)}`, true);
+    try {
+      const result = await options.call(plugin.path, params);
+      const body = JSON.stringify(result);
+      if (body.length <= MAX_OUTPUT_CHARS) return text(body);
+      return text(
+        body.slice(0, MAX_OUTPUT_CHARS) + `
+
+[Cut at ${MAX_OUTPUT_CHARS} of ${body.length} characters - ask for fewer results (max_results) or a narrower filter to get the whole answer.]`
+      );
+    } catch (err) {
+      return text(options.errorText(err), true);
+    }
+  }
   server2.registerTool(
     "list_plugins",
     {
@@ -5604,32 +5662,324 @@ function registerPluginTools(server2, options) {
         params: z.record(z.string(), z.union([z.string(), z.number(), z.boolean(), z.array(z.union([z.string(), z.number()]))])).optional().describe(`The plugin's parameters, e.g. {"query": "coffee maker", "max_results": 20}.`)
       }
     },
+    async (args) => run(BY_NAME.get(args.name), args.params ?? {})
+  );
+  for (const name of FEATURED_PLUGINS) {
+    const plugin = BY_NAME.get(name);
+    if (!plugin) continue;
+    const shape = Object.fromEntries(plugin.params.map((q) => [q.name, paramType(q)]));
+    server2.registerTool(
+      plugin.name,
+      {
+        title: plugin.title,
+        annotations: { title: plugin.title, ...annotations },
+        description: `${plugin.description} Returns JSON. Billed like an API request; listing plugins bill one unit per result page they fetch, so keep max_results modest.`,
+        inputSchema: shape
+      },
+      async (args) => {
+        const raw = Object.fromEntries(
+          Object.entries(args).filter(([, v]) => v !== void 0)
+        );
+        return run(plugin, raw);
+      }
+    );
+  }
+}
+
+// src/reading-tools.ts
+import { z as z2 } from "zod";
+
+// src/markdown.ts
+import { parseHTML } from "linkedom";
+import TurndownService from "turndown";
+import { gfm } from "turndown-plugin-gfm";
+var ALWAYS_DROP = [
+  "script",
+  "style",
+  "noscript",
+  "template",
+  "svg",
+  "canvas",
+  "iframe",
+  "object",
+  "embed",
+  "link",
+  "meta",
+  "head",
+  "button",
+  "input",
+  "select",
+  "textarea",
+  "dialog",
+  "[hidden]",
+  '[aria-hidden="true"]'
+].join(",");
+var CHROME = ["nav", "footer", "aside", '[role="navigation"]', '[role="contentinfo"]', '[role="complementary"]'].join(",");
+var MAIN_MIN_SHARE = 0.3;
+function textLength(el) {
+  return (el?.textContent ?? "").replace(/\s+/g, " ").trim().length;
+}
+function absolute(href, base) {
+  try {
+    const url = new URL(href, base);
+    return url.protocol === "http:" || url.protocol === "https:" ? url.href : null;
+  } catch {
+    return null;
+  }
+}
+function converter() {
+  const td = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+    bulletListMarker: "-",
+    emDelimiter: "_"
+  });
+  td.use(gfm);
+  td.addRule("images", {
+    filter: "img",
+    replacement: (_content, node) => {
+      const el = node;
+      const alt = (el.getAttribute("alt") || "").replace(/\s+/g, " ").trim();
+      const src = el.getAttribute("src") || "";
+      return alt && /^https?:/i.test(src) ? `![${alt}](${src})` : "";
+    }
+  });
+  td.addRule("flatCells", {
+    filter: ["th", "td"],
+    replacement: (content, node) => {
+      const cell = node;
+      const first = Array.prototype.indexOf.call(cell.parentNode.childNodes, cell) === 0;
+      const text3 = content.replace(/\s*\n+\s*/g, " ").replace(/\|/g, "\\|").trim();
+      return `${first ? "| " : " "}${text3} |`;
+    }
+  });
+  return td;
+}
+function normalizeTables(document) {
+  for (const table of [...document.querySelectorAll("table")].reverse()) {
+    const rows = [...table.querySelectorAll("tr")].filter((tr) => tr.closest("table") === table).map((tr) => [...tr.children].filter((c) => c.tagName === "TD" || c.tagName === "TH")).filter((cells) => cells.length > 0);
+    if (rows.length === 0) {
+      table.remove();
+      continue;
+    }
+    if (rows.length === 1 || Math.max(...rows.map((cells) => cells.length)) === 1) {
+      const blocks = document.createElement("div");
+      for (const [cell] of rows) {
+        const block = document.createElement("div");
+        block.innerHTML = cell.innerHTML;
+        blocks.appendChild(block);
+      }
+      table.replaceWith(blocks);
+      continue;
+    }
+    const clean = document.createElement("table");
+    rows.forEach((cells, i) => {
+      const section = document.createElement(i === 0 ? "thead" : "tbody");
+      const tr = document.createElement("tr");
+      for (const cell of cells) {
+        const out = document.createElement(i === 0 ? "th" : "td");
+        out.innerHTML = cell.innerHTML.replace(/\s+/g, " ");
+        tr.appendChild(out);
+      }
+      if (i === 0 || !clean.lastElementChild || clean.lastElementChild.tagName !== "TBODY") {
+        section.appendChild(tr);
+        clean.appendChild(section);
+      } else {
+        clean.lastElementChild.appendChild(tr);
+      }
+    });
+    table.replaceWith(clean);
+  }
+}
+function htmlToMarkdown(html, url, options) {
+  if (!/<[a-z!][\s\S]*>/i.test(html)) return { title: "", markdown: html.trim() };
+  const { document } = parseHTML(html);
+  const title = (document.querySelector("title")?.textContent ?? "").replace(/\s+/g, " ").trim();
+  for (const el of document.querySelectorAll(ALWAYS_DROP)) el.remove();
+  for (const a of document.querySelectorAll("a[href]")) {
+    const href = absolute(a.getAttribute("href") || "", url);
+    if (href) a.setAttribute("href", href);
+    else a.removeAttribute("href");
+  }
+  for (const img of document.querySelectorAll("img")) {
+    const src = absolute(img.getAttribute("src") || "", url);
+    if (src) img.setAttribute("src", src);
+    else img.removeAttribute("src");
+  }
+  const body = document.body ?? document.documentElement;
+  let root = body;
+  if (options.mainOnly) {
+    const main2 = document.querySelector('main, [role="main"]');
+    if (main2 && textLength(main2) >= MAIN_MIN_SHARE * textLength(body)) {
+      root = main2;
+    } else {
+      for (const el of document.querySelectorAll(CHROME)) el.remove();
+      for (const header of document.querySelectorAll('header, [role="banner"]')) {
+        if (!header.closest("article, main")) header.remove();
+      }
+    }
+  }
+  normalizeTables(document);
+  let converted;
+  try {
+    converted = converter().turndown(root.innerHTML);
+  } catch {
+    converted = (root.textContent ?? "").replace(/[ \t]+/g, " ").replace(/\s*\n\s*/g, "\n");
+  }
+  const markdown = converted.replace(/\[\s*\]\([^)]*\)/g, "").replace(/^(\s*)([-*+]|\d+\.) {2,}/gm, "$1$2 ").replace(/(\)|\*\*|_) ([.,;:!?])/g, "$1$2").replace(/[ \t]+$/gm, "").replace(/\n{3,}/g, "\n\n").trim();
+  return { title, markdown };
+}
+function capText(text3, max, hint) {
+  if (text3.length <= max) return text3;
+  return `${text3.slice(0, max)}
+
+[Cut at ${max} of ${text3.length} characters. ${hint}]`;
+}
+
+// src/reading-tools.ts
+var PAGE_MAX_CHARS = 1e5;
+var SEARCH_PAGE_MAX_CHARS = 15e3;
+var SEARCH_DEFAULT_RESULTS = 3;
+var SEARCH_MAX_RESULTS = 5;
+var SEARCH_PAGE_DEADLINE_MS = 12e4;
+function text2(body, isError = false) {
+  return isError ? { content: [{ type: "text", text: body }], isError } : { content: [{ type: "text", text: body }] };
+}
+function pageMarkdown(url, html, mainOnly, max) {
+  const page = htmlToMarkdown(html, url, { mainOnly });
+  const head = page.title ? `URL: ${url}
+Title: ${page.title}` : `URL: ${url}`;
+  return capText(`${head}
+
+${page.markdown}`, max, "Use fetch_html for the whole page.");
+}
+function organicResults(serp) {
+  const organic = serp?.organic;
+  if (!Array.isArray(organic)) return [];
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const item of organic) {
+    const url = typeof item?.url === "string" ? item.url : "";
+    if (!/^https?:\/\//i.test(url) || seen.has(url)) continue;
+    seen.add(url);
+    out.push({
+      title: typeof item.title === "string" ? item.title : url,
+      url,
+      description: typeof item.description === "string" ? item.description : ""
+    });
+  }
+  return out;
+}
+function aiOverview(serp) {
+  const overview = serp?.aiOverview;
+  if (!overview || typeof overview.text !== "string" || !overview.text.trim()) return null;
+  const sources = Array.isArray(overview.sources) ? overview.sources.filter((s) => typeof s === "string") : [];
+  return sources.length ? `${overview.text.trim()}
+
+Sources: ${sources.join(", ")}` : overview.text.trim();
+}
+function withDeadline(work, ms) {
+  let timer;
+  const deadline = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(new Error(`no answer within ${ms / 1e3} s`)), ms);
+  });
+  return Promise.race([work, deadline]).finally(() => clearTimeout(timer));
+}
+var proxyCountry = z2.string().length(2).optional().describe("Optional ISO country code to route through, e.g. 'US'.");
+function registerReadingTools(server2, options) {
+  const annotations = { readOnlyHint: true, openWorldHint: true, destructiveHint: false };
+  server2.registerTool(
+    "fetch_markdown",
+    {
+      title: "Fetch page as Markdown",
+      annotations: { title: "Fetch page as Markdown", ...annotations },
+      description: "Fetch a web page in a real browser through the ScrapeUnblocker API (https://docs.scrapeunblocker.com) and return it as clean Markdown: the main content with absolute links, without scripts, styles, navigation or footer. Use it to read, quote or summarise a page; it is usually 2-25% the size of the HTML. Works on JavaScript-rendered pages and on pages that refuse ordinary fetches (403/429, captcha). Use fetch_html instead when you need the markup, selectors or browser steps. Billed as one request.",
+      inputSchema: {
+        url: z2.string().url().describe("The absolute URL to read (http/https)."),
+        proxy_country: proxyCountry,
+        wait_method: z2.enum(["css", "js"]).optional().describe("Optional render-wait: 'css' selector or 'js' expression."),
+        wait_value: z2.string().optional().describe("The selector/expression paired with wait_method."),
+        sleep_seconds: z2.number().positive().optional().describe("Extra seconds to wait after load."),
+        main_content_only: z2.boolean().optional().describe("Drop navigation, page header, footer and sidebars (default true). False keeps the whole page.")
+      }
+    },
     async (args) => {
       const blocked = options.blocked?.();
       if (blocked) return blocked;
-      const plugin = BY_NAME.get(args.name);
-      const params = prepare(plugin, args.params ?? {});
-      if (typeof params === "string") return text(`${params}
-
-${details(plugin)}`, true);
       try {
-        const result = await options.call(plugin.path, params);
-        const body = JSON.stringify(result);
-        if (body.length <= MAX_OUTPUT_CHARS) return text(body);
-        return text(
-          body.slice(0, MAX_OUTPUT_CHARS) + `
+        const html = await options.fetchHtml(args.url, {
+          proxyCountry: args.proxy_country,
+          method: args.wait_method,
+          value: args.wait_value,
+          timeSleep: args.sleep_seconds
+        });
+        return text2(pageMarkdown(args.url, html, args.main_content_only ?? true, PAGE_MAX_CHARS));
+      } catch (err) {
+        return options.fetchError(args.url, err);
+      }
+    }
+  );
+  server2.registerTool(
+    "search_and_read",
+    {
+      title: "Search Google and read the top results",
+      annotations: { title: "Search Google and read the top results", ...annotations },
+      description: "Search Google through the ScrapeUnblocker API (https://docs.scrapeunblocker.com) and read the top results in one call: returns Google's AI Overview when there is one, each of the first `results` pages (default 3, max 5) as Markdown, and the other organic results as title, URL and snippet. Use it to answer a question from the live web. Billed as one request for the search plus one per page read.",
+      inputSchema: {
+        keyword: z2.string().min(1).describe("The search query."),
+        proxy_country: z2.string().length(2).optional().describe("Optional ISO country code to search from and read the pages through, e.g. 'US'."),
+        results: z2.number().int().min(1).max(SEARCH_MAX_RESULTS).optional().describe(`How many top results to read (default ${SEARCH_DEFAULT_RESULTS}, max ${SEARCH_MAX_RESULTS}).`)
+      }
+    },
+    async (args) => {
+      const blocked = options.blocked?.();
+      if (blocked) return blocked;
+      let serp;
+      try {
+        serp = await options.serp(args.keyword, { proxyCountry: args.proxy_country });
+      } catch (err) {
+        return text2(options.errorText(err), true);
+      }
+      const organic = organicResults(serp);
+      const overview = aiOverview(serp);
+      if (!organic.length && !overview) {
+        return text2(`Google returned no organic results for "${args.keyword}". The search was billed as one request.`);
+      }
+      const toRead = organic.slice(0, args.results ?? SEARCH_DEFAULT_RESULTS);
+      const pages = await Promise.allSettled(
+        toRead.map(
+          (r) => withDeadline(options.fetchHtml(r.url, { proxyCountry: args.proxy_country }), SEARCH_PAGE_DEADLINE_MS)
+        )
+      );
+      const parts = [
+        `Google results for "${args.keyword}": ${toRead.length} read below. Billed as one search plus one request per page read.`
+      ];
+      if (overview) parts.push(`## Google AI Overview
 
-[Cut at ${MAX_OUTPUT_CHARS} of ${body.length} characters - ask for fewer results (max_results) or a narrower filter to get the whole answer.]`
+${overview}`);
+      toRead.forEach((r, i) => {
+        const page = pages[i];
+        const body = page.status === "fulfilled" ? pageMarkdown(r.url, page.value, true, SEARCH_PAGE_MAX_CHARS) : `URL: ${r.url}
+
+Could not read this page: ${options.errorText(page.reason)}
+Snippet: ${r.description}`;
+        parts.push(`## ${i + 1}. ${r.title}
+
+${body}`);
+      });
+      const rest = organic.slice(toRead.length);
+      if (rest.length) {
+        parts.push(
+          "## Other results\n\n" + rest.map((r) => `- [${r.title}](${r.url}): ${r.description}`).join("\n")
         );
-      } catch (err) {
-        return text(options.errorText(err), true);
       }
+      return text2(parts.join("\n\n"));
     }
   );
 }
 
 // src/index.ts
-var VERSION = "0.3.5";
+var VERSION = "0.5.0";
 var DEFAULT_BASE_URL = "https://api.scrapeunblocker.com";
 var API_KEY_HEADER = "x-scrapeunblocker-key";
 function resolveApiKey() {
@@ -5707,11 +6057,11 @@ async function getPageSourceRaw(params) {
       Accept: "*/*"
     }
   });
-  const text2 = await response.text();
+  const text3 = await response.text();
   return {
     status: response.status,
     ok: response.ok,
-    text: text2,
+    text: text3,
     originStatus: response.headers.get("x-origin-status")
   };
 }
@@ -5719,8 +6069,8 @@ var server = new McpServer({
   name: "scrapeunblocker",
   version: VERSION
 });
-var selectorTypeSchema = z2.enum(["css", "xPath", "className", "tagName"]).describe("How `selector` is interpreted. Defaults to 'css'.");
-var pressableKeySchema = z2.enum([
+var selectorTypeSchema = z3.enum(["css", "xPath", "className", "tagName"]).describe("How `selector` is interpreted. Defaults to 'css'.");
+var pressableKeySchema = z3.enum([
   "Enter",
   "Tab",
   "Escape",
@@ -5736,71 +6086,72 @@ var pressableKeySchema = z2.enum([
   "PageUp",
   "PageDown"
 ]);
-var stepSchema = z2.discriminatedUnion("action", [
-  z2.object({
-    action: z2.literal("wait_for"),
-    selector: z2.string().describe("Selector to wait for (until present)."),
+var stepSchema = z3.discriminatedUnion("action", [
+  z3.object({
+    action: z3.literal("wait_for"),
+    selector: z3.string().describe("Selector to wait for (until present)."),
     selector_type: selectorTypeSchema.optional(),
-    timeout_ms: z2.number().int().positive().optional()
+    timeout_ms: z3.number().int().positive().optional()
   }).describe("Wait until an element matching `selector` exists."),
-  z2.object({
-    action: z2.literal("wait_for_text"),
-    value: z2.string().describe("Text to wait for anywhere on the page."),
-    timeout_ms: z2.number().int().positive().optional()
+  z3.object({
+    action: z3.literal("wait_for_text"),
+    value: z3.string().describe("Text to wait for anywhere on the page."),
+    timeout_ms: z3.number().int().positive().optional()
   }).describe("Wait until the given text appears on the page."),
-  z2.object({
-    action: z2.literal("wait"),
-    value: z2.number().int().nonnegative().describe("Fixed pause in milliseconds.")
+  z3.object({
+    action: z3.literal("wait"),
+    value: z3.number().int().nonnegative().describe("Fixed pause in milliseconds.")
   }).describe("Pause for a fixed number of milliseconds."),
-  z2.object({
-    action: z2.literal("click"),
-    selector: z2.string().describe("Selector of the element to click."),
+  z3.object({
+    action: z3.literal("click"),
+    selector: z3.string().describe("Selector of the element to click."),
     selector_type: selectorTypeSchema.optional(),
-    timeout_ms: z2.number().int().positive().optional()
+    timeout_ms: z3.number().int().positive().optional()
   }).describe("Click the element matching `selector`."),
-  z2.object({
-    action: z2.literal("type"),
-    selector: z2.string().describe("Selector of the field to type into."),
+  z3.object({
+    action: z3.literal("type"),
+    selector: z3.string().describe("Selector of the field to type into."),
     selector_type: selectorTypeSchema.optional(),
-    value: z2.string().describe("Text to type (entered human-like)."),
-    clear: z2.boolean().optional().describe("Clear the field before typing."),
-    timeout_ms: z2.number().int().positive().optional()
+    value: z3.string().describe("Text to type (entered human-like)."),
+    clear: z3.boolean().optional().describe("Clear the field before typing."),
+    timeout_ms: z3.number().int().positive().optional()
   }).describe("Type text into an input, character by character."),
-  z2.object({
-    action: z2.literal("select"),
-    selector: z2.string().describe("Selector of the <select> element."),
+  z3.object({
+    action: z3.literal("select"),
+    selector: z3.string().describe("Selector of the <select> element."),
     selector_type: selectorTypeSchema.optional(),
-    value: z2.string().describe("The option value to select."),
-    timeout_ms: z2.number().int().positive().optional()
+    value: z3.string().describe("The option value to select."),
+    timeout_ms: z3.number().int().positive().optional()
   }).describe("Choose an option in a <select> dropdown by value."),
-  z2.object({
-    action: z2.literal("press_key"),
+  z3.object({
+    action: z3.literal("press_key"),
     value: pressableKeySchema.describe("The key to press.")
   }).describe("Press a single keyboard key."),
-  z2.object({
-    action: z2.literal("scroll"),
-    value: z2.union([z2.literal("bottom"), z2.number().int()]).describe("'bottom' to scroll to the end, or a pixel offset.")
+  z3.object({
+    action: z3.literal("scroll"),
+    value: z3.union([z3.literal("bottom"), z3.number().int()]).describe("'bottom' to scroll to the end, or a pixel offset.")
   }).describe("Scroll the page to the bottom or by a pixel amount.")
 ]);
 server.registerTool(
   "fetch_html",
   {
     title: "Fetch page HTML",
+    annotations: { title: "Fetch page HTML", readOnlyHint: true, openWorldHint: true, destructiveHint: false },
     description: "Fetch the fully rendered HTML of any web page through ScrapeUnblocker, bypassing anti-bot protection (Cloudflare, DataDome, PerimeterX, Akamai, Shape). Use this when a normal fetch is blocked (403/429, captcha, 'access denied') or when the page needs a real browser to render. Returns the raw HTML as text.\n\nFor pages that need interaction (accept a cookie banner, click a tab, type into a search box, scroll to trigger lazy loading) pass `steps`: an ordered list of browser actions run in a real browser AFTER the page loads, then the resulting HTML is returned. Workflow: first call the `list_elements` tool to discover the real selectors on the page, then build `steps` against them. Steps are NOT idempotent - they run once per call. If a step fails, this tool returns which step failed, why, and the page HTML at that moment so you can fix the selector and retry.",
     inputSchema: {
-      url: z2.string().url().describe("The absolute URL to fetch (http/https)."),
-      proxy_country: z2.string().length(2).optional().describe(
+      url: z3.string().url().describe("The absolute URL to fetch (http/https)."),
+      proxy_country: z3.string().length(2).optional().describe(
         "Optional ISO 3166-1 alpha-2 country code to route through, e.g. 'US', 'GB', 'DE'."
       ),
-      wait_method: z2.enum(["css", "js"]).optional().describe(
+      wait_method: z3.enum(["css", "js"]).optional().describe(
         "Optional render-wait strategy: 'css' waits for a selector, 'js' waits for a JS expression to be truthy."
       ),
-      wait_value: z2.string().optional().describe(
+      wait_value: z3.string().optional().describe(
         `The CSS selector or JS expression paired with wait_method (e.g. '#price' or 'document.readyState==="complete"').`
       ),
-      method_timeout_seconds: z2.number().int().positive().optional().describe("Cap in seconds for the render-wait method."),
-      sleep_seconds: z2.number().positive().optional().describe("Extra seconds to wait after load before capturing the HTML."),
-      steps: z2.array(stepSchema).optional().describe(
+      method_timeout_seconds: z3.number().int().positive().optional().describe("Cap in seconds for the render-wait method."),
+      sleep_seconds: z3.number().positive().optional().describe("Extra seconds to wait after load before capturing the HTML."),
+      steps: z3.array(stepSchema).optional().describe(
         "Optional ordered browser actions to run in a real browser after the page loads, before the HTML is captured. Use `list_elements` first to find selectors. Runs once (not idempotent)."
       )
     }
@@ -5871,10 +6222,11 @@ server.registerTool(
   "list_elements",
   {
     title: "List page elements (selector discovery)",
+    annotations: { title: "List page elements (selector discovery)", readOnlyHint: true, openWorldHint: true, destructiveHint: false },
     description: "Load a page through ScrapeUnblocker and return a JSON list of its notable elements (links, inputs, buttons, selects, etc.) with a ready-to-use `selector` for each, plus tag, text and useful attributes (name, id, type, placeholder, aria_label, href, ...). Read-only and does not interact with the page. This is the discovery half of interactive scraping: call `list_elements` to find the selectors you need, then pass matching `steps` to `fetch_html` to click/type/select and capture the resulting HTML.",
     inputSchema: {
-      url: z2.string().url().describe("The absolute URL whose elements you want to list."),
-      proxy_country: z2.string().length(2).optional().describe("Optional ISO country code to route through, e.g. 'US'.")
+      url: z3.string().url().describe("The absolute URL whose elements you want to list."),
+      proxy_country: z3.string().length(2).optional().describe("Optional ISO country code to route through, e.g. 'US'.")
     }
   },
   async (args) => {
@@ -5914,11 +6266,12 @@ server.registerTool(
   "fetch_parsed",
   {
     title: "Fetch AI-parsed page data",
+    annotations: { title: "Fetch AI-parsed page data", readOnlyHint: true, openWorldHint: true, destructiveHint: false },
     description: "Fetch a web page through ScrapeUnblocker and return AI-parsed structured JSON instead of raw HTML (e.g. product details, article content). Best for extracting fields from product, listing or article pages without writing your own HTML parsing. If the page holds no structured data, the result says so and returns the rendered HTML instead (billed like fetch_html).",
     inputSchema: {
-      url: z2.string().url().describe("The absolute URL to fetch and parse."),
-      proxy_country: z2.string().length(2).optional().describe("Optional ISO country code to route through, e.g. 'US'."),
-      rules_hint: z2.string().optional().describe(
+      url: z3.string().url().describe("The absolute URL to fetch and parse."),
+      proxy_country: z3.string().length(2).optional().describe("Optional ISO country code to route through, e.g. 'US'."),
+      rules_hint: z3.string().optional().describe(
         "Optional natural-language hint about what to extract, to guide parsing."
       )
     }
@@ -5945,11 +6298,12 @@ server.registerTool(
   "google_search",
   {
     title: "Google search results",
+    annotations: { title: "Google search results", readOnlyHint: true, openWorldHint: true, destructiveHint: false },
     description: "Run a Google search through ScrapeUnblocker and return the organic results as structured JSON. Use this to discover URLs before fetching them.",
     inputSchema: {
-      keyword: z2.string().min(1).describe("The search query."),
-      proxy_country: z2.string().length(2).optional().describe("Optional ISO country code to search from, e.g. 'US'."),
-      pages_to_check: z2.number().int().positive().max(10).optional().describe("How many result pages to collect (default 1).")
+      keyword: z3.string().min(1).describe("The search query."),
+      proxy_country: z3.string().length(2).optional().describe("Optional ISO country code to search from, e.g. 'US'."),
+      pages_to_check: z3.number().int().positive().max(10).optional().describe("How many result pages to collect (default 1).")
     }
   },
   async (args) => {
@@ -5966,6 +6320,12 @@ server.registerTool(
     }
   }
 );
+registerReadingTools(server, {
+  fetchHtml: (url, options) => client().getPageSource(url, options),
+  serp: (keyword, options) => client().serp(keyword, options),
+  fetchError: errorResult,
+  errorText
+});
 registerPluginTools(server, {
   call: (path, params) => client().plugin(path, params),
   errorText

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrapeunblocker-mcp",
-  "version": "0.3.5",
+  "version": "0.5.0",
   "mcpName": "com.scrapeunblocker/scrapeunblocker-mcp",
   "description": "Model Context Protocol (MCP) server for ScrapeUnblocker - lets Claude fetch any web page's HTML (or AI-parsed JSON) and run 100+ site plugins that return structured JSON, through the ScrapeUnblocker anti-bot API, using your own API key.",
   "keywords": [
@@ -43,11 +43,15 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.32.1",
+    "linkedom": "^0.18.13",
     "scrapeunblocker": "^0.10.0",
+    "turndown": "^7.2.4",
+    "turndown-plugin-gfm": "^1.0.2",
     "zod": "^3.23.8"
   },
   "devDependencies": {
     "@types/node": "^20.19.43",
+    "@types/turndown": "^5.0.6",
     "tsup": "^8.0.0",
     "tsx": "^4.16.0",
     "typescript": "^5.4.0"