scrapeunblocker-mcp 0.6.0 → 0.7.0

package/README.md

@@ -156,6 +156,7 @@ SCRAPEUNBLOCKER_KEY=your_api_key_here npx -y scrapeunblocker-mcp
 npm install
 npm run build      # bundles to dist/ with tsup
 npm run typecheck
+npm test           # builds, then runs the tools against a local stand-in API
 SCRAPEUNBLOCKER_KEY=... node dist/index.js   # run the server
 ```
 

package/dist/index.js

@@ -4,7 +4,7 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z as z3 } from "zod";
-import { APIError, ScrapeUnblockerClient } from "scrapeunblocker";
+import { APIError, ScrapeUnblockerClient, errorForStatus } from "scrapeunblocker";
 
 // src/plugin-tools.ts
 import { z } from "zod";
@@ -5688,155 +5688,6 @@ ${details(plugin)}`, true);
 
 // src/reading-tools.ts
 import { z as z2 } from "zod";
-
-// src/markdown.ts
-import { parseHTML } from "linkedom";
-import TurndownService from "turndown";
-import { gfm } from "turndown-plugin-gfm";
-var ALWAYS_DROP = [
-  "script",
-  "style",
-  "noscript",
-  "template",
-  "svg",
-  "canvas",
-  "iframe",
-  "object",
-  "embed",
-  "link",
-  "meta",
-  "head",
-  "button",
-  "input",
-  "select",
-  "textarea",
-  "dialog",
-  "[hidden]",
-  '[aria-hidden="true"]'
-].join(",");
-var CHROME = ["nav", "footer", "aside", '[role="navigation"]', '[role="contentinfo"]', '[role="complementary"]'].join(",");
-var MAIN_MIN_SHARE = 0.3;
-function textLength(el) {
-  return (el?.textContent ?? "").replace(/\s+/g, " ").trim().length;
-}
-function absolute(href, base) {
-  try {
-    const url = new URL(href, base);
-    return url.protocol === "http:" || url.protocol === "https:" ? url.href : null;
-  } catch {
-    return null;
-  }
-}
-function converter() {
-  const td = new TurndownService({
-    headingStyle: "atx",
-    codeBlockStyle: "fenced",
-    bulletListMarker: "-",
-    emDelimiter: "_"
-  });
-  td.use(gfm);
-  td.addRule("images", {
-    filter: "img",
-    replacement: (_content, node) => {
-      const el = node;
-      const alt = (el.getAttribute("alt") || "").replace(/\s+/g, " ").trim();
-      const src = el.getAttribute("src") || "";
-      return alt && /^https?:/i.test(src) ? `![${alt}](${src})` : "";
-    }
-  });
-  td.addRule("flatCells", {
-    filter: ["th", "td"],
-    replacement: (content, node) => {
-      const cell = node;
-      const first = Array.prototype.indexOf.call(cell.parentNode.childNodes, cell) === 0;
-      const text3 = content.replace(/\s*\n+\s*/g, " ").replace(/\|/g, "\\|").trim();
-      return `${first ? "| " : " "}${text3} |`;
-    }
-  });
-  return td;
-}
-function normalizeTables(document) {
-  for (const table of [...document.querySelectorAll("table")].reverse()) {
-    const rows = [...table.querySelectorAll("tr")].filter((tr) => tr.closest("table") === table).map((tr) => [...tr.children].filter((c) => c.tagName === "TD" || c.tagName === "TH")).filter((cells) => cells.length > 0);
-    if (rows.length === 0) {
-      table.remove();
-      continue;
-    }
-    if (rows.length === 1 || Math.max(...rows.map((cells) => cells.length)) === 1) {
-      const blocks = document.createElement("div");
-      for (const [cell] of rows) {
-        const block = document.createElement("div");
-        block.innerHTML = cell.innerHTML;
-        blocks.appendChild(block);
-      }
-      table.replaceWith(blocks);
-      continue;
-    }
-    const clean = document.createElement("table");
-    rows.forEach((cells, i) => {
-      const section = document.createElement(i === 0 ? "thead" : "tbody");
-      const tr = document.createElement("tr");
-      for (const cell of cells) {
-        const out = document.createElement(i === 0 ? "th" : "td");
-        out.innerHTML = cell.innerHTML.replace(/\s+/g, " ");
-        tr.appendChild(out);
-      }
-      if (i === 0 || !clean.lastElementChild || clean.lastElementChild.tagName !== "TBODY") {
-        section.appendChild(tr);
-        clean.appendChild(section);
-      } else {
-        clean.lastElementChild.appendChild(tr);
-      }
-    });
-    table.replaceWith(clean);
-  }
-}
-function htmlToMarkdown(html, url, options) {
-  if (!/<[a-z!][\s\S]*>/i.test(html)) return { title: "", markdown: html.trim() };
-  const { document } = parseHTML(html);
-  const title = (document.querySelector("title")?.textContent ?? "").replace(/\s+/g, " ").trim();
-  for (const el of document.querySelectorAll(ALWAYS_DROP)) el.remove();
-  for (const a of document.querySelectorAll("a[href]")) {
-    const href = absolute(a.getAttribute("href") || "", url);
-    if (href) a.setAttribute("href", href);
-    else a.removeAttribute("href");
-  }
-  for (const img of document.querySelectorAll("img")) {
-    const src = absolute(img.getAttribute("src") || "", url);
-    if (src) img.setAttribute("src", src);
-    else img.removeAttribute("src");
-  }
-  const body = document.body ?? document.documentElement;
-  let root = body;
-  if (options.mainOnly) {
-    const main2 = document.querySelector('main, [role="main"]');
-    if (main2 && textLength(main2) >= MAIN_MIN_SHARE * textLength(body)) {
-      root = main2;
-    } else {
-      for (const el of document.querySelectorAll(CHROME)) el.remove();
-      for (const header of document.querySelectorAll('header, [role="banner"]')) {
-        if (!header.closest("article, main")) header.remove();
-      }
-    }
-  }
-  normalizeTables(document);
-  let converted;
-  try {
-    converted = converter().turndown(root.innerHTML);
-  } catch {
-    converted = (root.textContent ?? "").replace(/[ \t]+/g, " ").replace(/\s*\n\s*/g, "\n");
-  }
-  const markdown = converted.replace(/\[\s*\]\([^)]*\)/g, "").replace(/^(\s*)([-*+]|\d+\.) {2,}/gm, "$1$2 ").replace(/(\)|\*\*|_) ([.,;:!?])/g, "$1$2").replace(/[ \t]+$/gm, "").replace(/\n{3,}/g, "\n\n").trim();
-  return { title, markdown };
-}
-function capText(text3, max, hint) {
-  if (text3.length <= max) return text3;
-  return `${text3.slice(0, max)}
-
-[Cut at ${max} of ${text3.length} characters. ${hint}]`;
-}
-
-// src/reading-tools.ts
 var PAGE_MAX_CHARS = 1e5;
 var SEARCH_PAGE_MAX_CHARS = 15e3;
 var SEARCH_DEFAULT_RESULTS = 3;
@@ -5845,13 +5696,27 @@ var SEARCH_PAGE_DEADLINE_MS = 12e4;
 function text2(body, isError = false) {
   return isError ? { content: [{ type: "text", text: body }], isError } : { content: [{ type: "text", text: body }] };
 }
-function pageMarkdown(url, html, mainOnly, max) {
-  const page = htmlToMarkdown(html, url, { mainOnly });
-  const head = page.title ? `URL: ${url}
-Title: ${page.title}` : `URL: ${url}`;
-  return capText(`${head}
+function markdownParams(url, options) {
+  return {
+    url,
+    markdown: true,
+    main_content_only: options.mainContentOnly,
+    proxy_country: options.proxyCountry,
+    method: options.method,
+    value: options.value,
+    time_sleep: options.timeSleep
+  };
+}
+function capText(text3, max, hint) {
+  if (text3.length <= max) return text3;
+  return `${text3.slice(0, max)}
+
+[Cut at ${max} of ${text3.length} characters. ${hint}]`;
+}
+function pageText(url, markdown, max) {
+  return capText(`URL: ${url}
 
-${page.markdown}`, max, "Use fetch_html for the whole page.");
+${markdown.trim()}`, max, "Use fetch_html for the whole page.");
 }
 function organicResults(serp) {
   const organic = serp?.organic;
@@ -5907,13 +5772,16 @@ function registerReadingTools(server2, options) {
       const blocked = options.blocked?.();
       if (blocked) return blocked;
       try {
-        const html = await options.fetchHtml(args.url, {
-          proxyCountry: args.proxy_country,
-          method: args.wait_method,
-          value: args.wait_value,
-          timeSleep: args.sleep_seconds
-        });
-        return text2(pageMarkdown(args.url, html, args.main_content_only ?? true, PAGE_MAX_CHARS));
+        const markdown = await options.getPageSource(
+          markdownParams(args.url, {
+            mainContentOnly: args.main_content_only ?? true,
+            proxyCountry: args.proxy_country,
+            method: args.wait_method,
+            value: args.wait_value,
+            timeSleep: args.sleep_seconds
+          })
+        );
+        return text2(pageText(args.url, markdown, PAGE_MAX_CHARS));
       } catch (err) {
         return options.fetchError(args.url, err);
       }
@@ -5948,7 +5816,10 @@ function registerReadingTools(server2, options) {
       const toRead = organic.slice(0, args.results ?? SEARCH_DEFAULT_RESULTS);
       const pages = await Promise.allSettled(
         toRead.map(
-          (r) => withDeadline(options.fetchHtml(r.url, { proxyCountry: args.proxy_country }), SEARCH_PAGE_DEADLINE_MS)
+          (r) => withDeadline(
+            options.getPageSource(markdownParams(r.url, { mainContentOnly: true, proxyCountry: args.proxy_country })),
+            SEARCH_PAGE_DEADLINE_MS
+          )
         )
       );
       const parts = [
@@ -5959,7 +5830,7 @@ function registerReadingTools(server2, options) {
 ${overview}`);
       toRead.forEach((r, i) => {
         const page = pages[i];
-        const body = page.status === "fulfilled" ? pageMarkdown(r.url, page.value, true, SEARCH_PAGE_MAX_CHARS) : `URL: ${r.url}
+        const body = page.status === "fulfilled" ? pageText(r.url, page.value, SEARCH_PAGE_MAX_CHARS) : `URL: ${r.url}
 
 Could not read this page: ${options.errorText(page.reason)}
 Snippet: ${r.description}`;
@@ -5979,7 +5850,7 @@ ${body}`);
 }
 
 // src/index.ts
-var VERSION = "0.6.0";
+var VERSION = "0.7.0";
 var DEFAULT_BASE_URL = "https://api.scrapeunblocker.com";
 var API_KEY_HEADER = "x-scrapeunblocker-key";
 function resolveApiKey() {
@@ -6040,22 +5911,27 @@ function errorResult(url, err) {
   }
   return { content: [{ type: "text", text: errorText(err) }], isError: true };
 }
-async function getPageSourceRaw(params) {
-  const apiKey = resolveApiKey();
-  if (!apiKey) throw new Error(MISSING_KEY_MESSAGE);
+function pageSourceUrl(params) {
   const query = new URLSearchParams();
   for (const [key, value] of Object.entries(params)) {
     if (value === void 0 || value === null) continue;
     query.set(key, String(value));
   }
-  const url = `${resolveBaseUrl()}/getPageSource?${query.toString()}`;
-  const response = await fetch(url, {
+  return `${resolveBaseUrl()}/getPageSource?${query.toString()}`;
+}
+function requestHeaders(apiKey) {
+  return {
+    [API_KEY_HEADER]: apiKey,
+    "User-Agent": `scrapeunblocker-mcp/${VERSION}`,
+    Accept: "*/*"
+  };
+}
+async function getPageSourceRaw(params) {
+  const apiKey = resolveApiKey();
+  if (!apiKey) throw new Error(MISSING_KEY_MESSAGE);
+  const response = await fetch(pageSourceUrl(params), {
     method: "POST",
-    headers: {
-      [API_KEY_HEADER]: apiKey,
-      "User-Agent": `scrapeunblocker-mcp/${VERSION}`,
-      Accept: "*/*"
-    }
+    headers: requestHeaders(apiKey)
   });
   const text3 = await response.text();
   return {
@@ -6065,6 +5941,42 @@ async function getPageSourceRaw(params) {
     originStatus: response.headers.get("x-origin-status")
   };
 }
+var READ_TIMEOUT_MS = 18e4;
+var READ_RETRIES = 2;
+var RETRYABLE = /* @__PURE__ */ new Set([429, 502, 503, 504]);
+function backoff(attempt) {
+  return new Promise((resolve) => setTimeout(resolve, Math.min(500 * 2 ** attempt, 8e3)));
+}
+async function readPageSource(params) {
+  const apiKey = resolveApiKey();
+  if (!apiKey) throw new Error(MISSING_KEY_MESSAGE);
+  const url = pageSourceUrl(params);
+  for (let attempt = 0; ; attempt++) {
+    let response;
+    try {
+      response = await fetch(url, {
+        method: "POST",
+        headers: requestHeaders(apiKey),
+        signal: AbortSignal.timeout(READ_TIMEOUT_MS)
+      });
+    } catch (err) {
+      if (err instanceof Error && err.name === "TimeoutError") {
+        throw new Error(`No answer from ScrapeUnblocker within ${READ_TIMEOUT_MS / 1e3} s.`);
+      }
+      if (attempt >= READ_RETRIES) throw new Error(`Could not reach ScrapeUnblocker: ${errorText(err)}`);
+      await backoff(attempt);
+      continue;
+    }
+    if (RETRYABLE.has(response.status) && attempt < READ_RETRIES) {
+      await response.body?.cancel();
+      await backoff(attempt);
+      continue;
+    }
+    const body = await response.text();
+    if (!response.ok) throw errorForStatus(response.status, body, response.headers);
+    return body;
+  }
+}
 var server = new McpServer({
   name: "scrapeunblocker",
   version: VERSION
@@ -6351,7 +6263,7 @@ server.registerTool(
   }
 );
 registerReadingTools(server, {
-  fetchHtml: (url, options) => client().getPageSource(url, options),
+  getPageSource: readPageSource,
   serp: (keyword, options) => client().serp(keyword, options),
   fetchError: errorResult,
   errorText

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrapeunblocker-mcp",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "mcpName": "com.scrapeunblocker/scrapeunblocker-mcp",
   "description": "Model Context Protocol (MCP) server for ScrapeUnblocker - lets Claude fetch any web page's HTML (or AI-parsed JSON) and run 100+ site plugins that return structured JSON, through the ScrapeUnblocker anti-bot API, using your own API key.",
   "keywords": [
@@ -37,21 +37,18 @@
   "scripts": {
     "build": "tsup",
     "typecheck": "tsc --noEmit",
+    "test": "npm run build && node --test test/*.test.mjs",
     "start": "node dist/index.js",
     "dev": "tsx src/index.ts",
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.32.1",
-    "linkedom": "^0.18.13",
     "scrapeunblocker": "^0.10.0",
-    "turndown": "^7.2.4",
-    "turndown-plugin-gfm": "^1.0.2",
     "zod": "^3.23.8"
   },
   "devDependencies": {
     "@types/node": "^20.19.43",
-    "@types/turndown": "^5.0.6",
     "tsup": "^8.0.0",
     "tsx": "^4.16.0",
     "typescript": "^5.4.0"