scrapeunblocker-mcp 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +9 -6
  2. package/dist/index.js +135 -193
  3. package/package.json +2 -5
package/README.md CHANGED
@@ -12,7 +12,8 @@ You bring your **own** API key. Nothing is shared or proxied through us.
12
12
 
13
13
  | Tool | What it does |
14
14
  |------|--------------|
15
- | `fetch_html` | Fetch the fully rendered HTML of a URL (optionally after running interactive browser `steps`). |
15
+ | `fetch_html` | Fetch the fully rendered HTML of a URL. |
16
+ | `browse_page` | Run interactive browser `steps` on a page (click, type, select, scroll, wait), then return the resulting HTML. Not read-only: it acts on the website. |
16
17
  | `list_elements` | List a page's notable elements with a ready-to-use selector for each (selector discovery). |
17
18
  | `fetch_parsed` | Fetch a page and return AI-parsed structured JSON. |
18
19
  | `google_search` | Run a Google search and return organic results as JSON. |
@@ -43,7 +44,7 @@ Each run is billed like an API request.
43
44
 
44
45
  Some pages only reveal what you need after you interact with them - accept a
45
46
  cookie banner, click a tab, type into a search box and submit, or scroll to
46
- trigger lazy loading. Pass an optional `steps` array to `fetch_html` and those
47
+ trigger lazy loading. Pass a `steps` array to `browse_page` and those
47
48
  actions run in a **real browser**, in order, after the page loads; the resulting
48
49
  HTML is then returned.
49
50
 
@@ -60,8 +61,9 @@ Available actions:
60
61
  | `press_key` | `value` (`Enter`, `Tab`, `Escape`, `Backspace`, `Delete`, `Space`, `Arrow*`, `Home`, `End`, `PageUp`, `PageDown`) |
61
62
  | `scroll` | `value` (`"bottom"` or a pixel offset) |
62
63
 
63
- Steps are **not idempotent** - they run once per call. If a step fails,
64
- `fetch_html` returns which step failed, why, and the page HTML at that moment so
64
+ Steps are **not idempotent** - they run once per call - and they act on the
65
+ website, so `browse_page` is the one tool not marked read-only. If a step fails,
66
+ `browse_page` returns which step failed, why, and the page HTML at that moment so
65
67
  you can fix the selector and retry.
66
68
 
67
69
  ### List elements (discover selectors first)
@@ -76,7 +78,7 @@ you can fix the selector and retry.
76
78
  ```
77
79
 
78
80
  The natural workflow is **discover, then act**: call `list_elements` to find the
79
- selectors you need, then pass matching `steps` to `fetch_html` to click/type/
81
+ selectors you need, then pass matching `steps` to `browse_page` to click/type/
80
82
  select and capture the resulting HTML.
81
83
 
82
84
  ## Get an API key
@@ -140,7 +142,7 @@ SCRAPEUNBLOCKER_KEY=your_api_key_here npx -y scrapeunblocker-mcp
140
142
 
141
143
  - "Fetch the HTML of https://www.example-shop.com/product/123 and list the price."
142
144
  - "This page keeps blocking me: <url>. Use fetch_html to get it."
143
- - "List the elements on <url>, then use fetch_html steps to type 'laptop' into the search box, press Enter, wait for the results, and give me the HTML."
145
+ - "List the elements on <url>, then use browse_page to type 'laptop' into the search box, press Enter, wait for the results, and give me the HTML."
144
146
  - "Search Google for 'best running shoes 2026' and give me the top 5 links."
145
147
  - "Read https://en.wikipedia.org/wiki/Web_scraping as Markdown and summarise it in five bullets."
146
148
  - "Search the web for the current population of Vilnius and cite the pages you read."
@@ -154,6 +156,7 @@ SCRAPEUNBLOCKER_KEY=your_api_key_here npx -y scrapeunblocker-mcp
154
156
  npm install
155
157
  npm run build # bundles to dist/ with tsup
156
158
  npm run typecheck
159
+ npm test # builds, then runs the tools against a local stand-in API
157
160
  SCRAPEUNBLOCKER_KEY=... node dist/index.js # run the server
158
161
  ```
159
162
 
package/dist/index.js CHANGED
@@ -4,7 +4,7 @@
4
4
  import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
5
5
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
6
6
  import { z as z3 } from "zod";
7
- import { APIError, ScrapeUnblockerClient } from "scrapeunblocker";
7
+ import { APIError, ScrapeUnblockerClient, errorForStatus } from "scrapeunblocker";
8
8
 
9
9
  // src/plugin-tools.ts
10
10
  import { z } from "zod";
@@ -5688,155 +5688,6 @@ ${details(plugin)}`, true);
5688
5688
 
5689
5689
  // src/reading-tools.ts
5690
5690
  import { z as z2 } from "zod";
5691
-
5692
- // src/markdown.ts
5693
- import { parseHTML } from "linkedom";
5694
- import TurndownService from "turndown";
5695
- import { gfm } from "turndown-plugin-gfm";
5696
- var ALWAYS_DROP = [
5697
- "script",
5698
- "style",
5699
- "noscript",
5700
- "template",
5701
- "svg",
5702
- "canvas",
5703
- "iframe",
5704
- "object",
5705
- "embed",
5706
- "link",
5707
- "meta",
5708
- "head",
5709
- "button",
5710
- "input",
5711
- "select",
5712
- "textarea",
5713
- "dialog",
5714
- "[hidden]",
5715
- '[aria-hidden="true"]'
5716
- ].join(",");
5717
- var CHROME = ["nav", "footer", "aside", '[role="navigation"]', '[role="contentinfo"]', '[role="complementary"]'].join(",");
5718
- var MAIN_MIN_SHARE = 0.3;
5719
- function textLength(el) {
5720
- return (el?.textContent ?? "").replace(/\s+/g, " ").trim().length;
5721
- }
5722
- function absolute(href, base) {
5723
- try {
5724
- const url = new URL(href, base);
5725
- return url.protocol === "http:" || url.protocol === "https:" ? url.href : null;
5726
- } catch {
5727
- return null;
5728
- }
5729
- }
5730
- function converter() {
5731
- const td = new TurndownService({
5732
- headingStyle: "atx",
5733
- codeBlockStyle: "fenced",
5734
- bulletListMarker: "-",
5735
- emDelimiter: "_"
5736
- });
5737
- td.use(gfm);
5738
- td.addRule("images", {
5739
- filter: "img",
5740
- replacement: (_content, node) => {
5741
- const el = node;
5742
- const alt = (el.getAttribute("alt") || "").replace(/\s+/g, " ").trim();
5743
- const src = el.getAttribute("src") || "";
5744
- return alt && /^https?:/i.test(src) ? `![${alt}](${src})` : "";
5745
- }
5746
- });
5747
- td.addRule("flatCells", {
5748
- filter: ["th", "td"],
5749
- replacement: (content, node) => {
5750
- const cell = node;
5751
- const first = Array.prototype.indexOf.call(cell.parentNode.childNodes, cell) === 0;
5752
- const text3 = content.replace(/\s*\n+\s*/g, " ").replace(/\|/g, "\\|").trim();
5753
- return `${first ? "| " : " "}${text3} |`;
5754
- }
5755
- });
5756
- return td;
5757
- }
5758
- function normalizeTables(document) {
5759
- for (const table of [...document.querySelectorAll("table")].reverse()) {
5760
- const rows = [...table.querySelectorAll("tr")].filter((tr) => tr.closest("table") === table).map((tr) => [...tr.children].filter((c) => c.tagName === "TD" || c.tagName === "TH")).filter((cells) => cells.length > 0);
5761
- if (rows.length === 0) {
5762
- table.remove();
5763
- continue;
5764
- }
5765
- if (rows.length === 1 || Math.max(...rows.map((cells) => cells.length)) === 1) {
5766
- const blocks = document.createElement("div");
5767
- for (const [cell] of rows) {
5768
- const block = document.createElement("div");
5769
- block.innerHTML = cell.innerHTML;
5770
- blocks.appendChild(block);
5771
- }
5772
- table.replaceWith(blocks);
5773
- continue;
5774
- }
5775
- const clean = document.createElement("table");
5776
- rows.forEach((cells, i) => {
5777
- const section = document.createElement(i === 0 ? "thead" : "tbody");
5778
- const tr = document.createElement("tr");
5779
- for (const cell of cells) {
5780
- const out = document.createElement(i === 0 ? "th" : "td");
5781
- out.innerHTML = cell.innerHTML.replace(/\s+/g, " ");
5782
- tr.appendChild(out);
5783
- }
5784
- if (i === 0 || !clean.lastElementChild || clean.lastElementChild.tagName !== "TBODY") {
5785
- section.appendChild(tr);
5786
- clean.appendChild(section);
5787
- } else {
5788
- clean.lastElementChild.appendChild(tr);
5789
- }
5790
- });
5791
- table.replaceWith(clean);
5792
- }
5793
- }
5794
- function htmlToMarkdown(html, url, options) {
5795
- if (!/<[a-z!][\s\S]*>/i.test(html)) return { title: "", markdown: html.trim() };
5796
- const { document } = parseHTML(html);
5797
- const title = (document.querySelector("title")?.textContent ?? "").replace(/\s+/g, " ").trim();
5798
- for (const el of document.querySelectorAll(ALWAYS_DROP)) el.remove();
5799
- for (const a of document.querySelectorAll("a[href]")) {
5800
- const href = absolute(a.getAttribute("href") || "", url);
5801
- if (href) a.setAttribute("href", href);
5802
- else a.removeAttribute("href");
5803
- }
5804
- for (const img of document.querySelectorAll("img")) {
5805
- const src = absolute(img.getAttribute("src") || "", url);
5806
- if (src) img.setAttribute("src", src);
5807
- else img.removeAttribute("src");
5808
- }
5809
- const body = document.body ?? document.documentElement;
5810
- let root = body;
5811
- if (options.mainOnly) {
5812
- const main2 = document.querySelector('main, [role="main"]');
5813
- if (main2 && textLength(main2) >= MAIN_MIN_SHARE * textLength(body)) {
5814
- root = main2;
5815
- } else {
5816
- for (const el of document.querySelectorAll(CHROME)) el.remove();
5817
- for (const header of document.querySelectorAll('header, [role="banner"]')) {
5818
- if (!header.closest("article, main")) header.remove();
5819
- }
5820
- }
5821
- }
5822
- normalizeTables(document);
5823
- let converted;
5824
- try {
5825
- converted = converter().turndown(root.innerHTML);
5826
- } catch {
5827
- converted = (root.textContent ?? "").replace(/[ \t]+/g, " ").replace(/\s*\n\s*/g, "\n");
5828
- }
5829
- const markdown = converted.replace(/\[\s*\]\([^)]*\)/g, "").replace(/^(\s*)([-*+]|\d+\.) {2,}/gm, "$1$2 ").replace(/(\)|\*\*|_) ([.,;:!?])/g, "$1$2").replace(/[ \t]+$/gm, "").replace(/\n{3,}/g, "\n\n").trim();
5830
- return { title, markdown };
5831
- }
5832
- function capText(text3, max, hint) {
5833
- if (text3.length <= max) return text3;
5834
- return `${text3.slice(0, max)}
5835
-
5836
- [Cut at ${max} of ${text3.length} characters. ${hint}]`;
5837
- }
5838
-
5839
- // src/reading-tools.ts
5840
5691
  var PAGE_MAX_CHARS = 1e5;
5841
5692
  var SEARCH_PAGE_MAX_CHARS = 15e3;
5842
5693
  var SEARCH_DEFAULT_RESULTS = 3;
@@ -5845,13 +5696,27 @@ var SEARCH_PAGE_DEADLINE_MS = 12e4;
5845
5696
  function text2(body, isError = false) {
5846
5697
  return isError ? { content: [{ type: "text", text: body }], isError } : { content: [{ type: "text", text: body }] };
5847
5698
  }
5848
- function pageMarkdown(url, html, mainOnly, max) {
5849
- const page = htmlToMarkdown(html, url, { mainOnly });
5850
- const head = page.title ? `URL: ${url}
5851
- Title: ${page.title}` : `URL: ${url}`;
5852
- return capText(`${head}
5699
+ function markdownParams(url, options) {
5700
+ return {
5701
+ url,
5702
+ markdown: true,
5703
+ main_content_only: options.mainContentOnly,
5704
+ proxy_country: options.proxyCountry,
5705
+ method: options.method,
5706
+ value: options.value,
5707
+ time_sleep: options.timeSleep
5708
+ };
5709
+ }
5710
+ function capText(text3, max, hint) {
5711
+ if (text3.length <= max) return text3;
5712
+ return `${text3.slice(0, max)}
5713
+
5714
+ [Cut at ${max} of ${text3.length} characters. ${hint}]`;
5715
+ }
5716
+ function pageText(url, markdown, max) {
5717
+ return capText(`URL: ${url}
5853
5718
 
5854
- ${page.markdown}`, max, "Use fetch_html for the whole page.");
5719
+ ${markdown.trim()}`, max, "Use fetch_html for the whole page.");
5855
5720
  }
5856
5721
  function organicResults(serp) {
5857
5722
  const organic = serp?.organic;
@@ -5907,13 +5772,16 @@ function registerReadingTools(server2, options) {
5907
5772
  const blocked = options.blocked?.();
5908
5773
  if (blocked) return blocked;
5909
5774
  try {
5910
- const html = await options.fetchHtml(args.url, {
5911
- proxyCountry: args.proxy_country,
5912
- method: args.wait_method,
5913
- value: args.wait_value,
5914
- timeSleep: args.sleep_seconds
5915
- });
5916
- return text2(pageMarkdown(args.url, html, args.main_content_only ?? true, PAGE_MAX_CHARS));
5775
+ const markdown = await options.getPageSource(
5776
+ markdownParams(args.url, {
5777
+ mainContentOnly: args.main_content_only ?? true,
5778
+ proxyCountry: args.proxy_country,
5779
+ method: args.wait_method,
5780
+ value: args.wait_value,
5781
+ timeSleep: args.sleep_seconds
5782
+ })
5783
+ );
5784
+ return text2(pageText(args.url, markdown, PAGE_MAX_CHARS));
5917
5785
  } catch (err) {
5918
5786
  return options.fetchError(args.url, err);
5919
5787
  }
@@ -5948,7 +5816,10 @@ function registerReadingTools(server2, options) {
5948
5816
  const toRead = organic.slice(0, args.results ?? SEARCH_DEFAULT_RESULTS);
5949
5817
  const pages = await Promise.allSettled(
5950
5818
  toRead.map(
5951
- (r) => withDeadline(options.fetchHtml(r.url, { proxyCountry: args.proxy_country }), SEARCH_PAGE_DEADLINE_MS)
5819
+ (r) => withDeadline(
5820
+ options.getPageSource(markdownParams(r.url, { mainContentOnly: true, proxyCountry: args.proxy_country })),
5821
+ SEARCH_PAGE_DEADLINE_MS
5822
+ )
5952
5823
  )
5953
5824
  );
5954
5825
  const parts = [
@@ -5959,7 +5830,7 @@ function registerReadingTools(server2, options) {
5959
5830
  ${overview}`);
5960
5831
  toRead.forEach((r, i) => {
5961
5832
  const page = pages[i];
5962
- const body = page.status === "fulfilled" ? pageMarkdown(r.url, page.value, true, SEARCH_PAGE_MAX_CHARS) : `URL: ${r.url}
5833
+ const body = page.status === "fulfilled" ? pageText(r.url, page.value, SEARCH_PAGE_MAX_CHARS) : `URL: ${r.url}
5963
5834
 
5964
5835
  Could not read this page: ${options.errorText(page.reason)}
5965
5836
  Snippet: ${r.description}`;
@@ -5979,7 +5850,7 @@ ${body}`);
5979
5850
  }
5980
5851
 
5981
5852
  // src/index.ts
5982
- var VERSION = "0.5.0";
5853
+ var VERSION = "0.7.0";
5983
5854
  var DEFAULT_BASE_URL = "https://api.scrapeunblocker.com";
5984
5855
  var API_KEY_HEADER = "x-scrapeunblocker-key";
5985
5856
  function resolveApiKey() {
@@ -6040,22 +5911,27 @@ function errorResult(url, err) {
6040
5911
  }
6041
5912
  return { content: [{ type: "text", text: errorText(err) }], isError: true };
6042
5913
  }
6043
- async function getPageSourceRaw(params) {
6044
- const apiKey = resolveApiKey();
6045
- if (!apiKey) throw new Error(MISSING_KEY_MESSAGE);
5914
+ function pageSourceUrl(params) {
6046
5915
  const query = new URLSearchParams();
6047
5916
  for (const [key, value] of Object.entries(params)) {
6048
5917
  if (value === void 0 || value === null) continue;
6049
5918
  query.set(key, String(value));
6050
5919
  }
6051
- const url = `${resolveBaseUrl()}/getPageSource?${query.toString()}`;
6052
- const response = await fetch(url, {
5920
+ return `${resolveBaseUrl()}/getPageSource?${query.toString()}`;
5921
+ }
5922
+ function requestHeaders(apiKey) {
5923
+ return {
5924
+ [API_KEY_HEADER]: apiKey,
5925
+ "User-Agent": `scrapeunblocker-mcp/${VERSION}`,
5926
+ Accept: "*/*"
5927
+ };
5928
+ }
5929
+ async function getPageSourceRaw(params) {
5930
+ const apiKey = resolveApiKey();
5931
+ if (!apiKey) throw new Error(MISSING_KEY_MESSAGE);
5932
+ const response = await fetch(pageSourceUrl(params), {
6053
5933
  method: "POST",
6054
- headers: {
6055
- [API_KEY_HEADER]: apiKey,
6056
- "User-Agent": `scrapeunblocker-mcp/${VERSION}`,
6057
- Accept: "*/*"
6058
- }
5934
+ headers: requestHeaders(apiKey)
6059
5935
  });
6060
5936
  const text3 = await response.text();
6061
5937
  return {
@@ -6065,6 +5941,42 @@ async function getPageSourceRaw(params) {
6065
5941
  originStatus: response.headers.get("x-origin-status")
6066
5942
  };
6067
5943
  }
5944
+ var READ_TIMEOUT_MS = 18e4;
5945
+ var READ_RETRIES = 2;
5946
+ var RETRYABLE = /* @__PURE__ */ new Set([429, 502, 503, 504]);
5947
+ function backoff(attempt) {
5948
+ return new Promise((resolve) => setTimeout(resolve, Math.min(500 * 2 ** attempt, 8e3)));
5949
+ }
5950
+ async function readPageSource(params) {
5951
+ const apiKey = resolveApiKey();
5952
+ if (!apiKey) throw new Error(MISSING_KEY_MESSAGE);
5953
+ const url = pageSourceUrl(params);
5954
+ for (let attempt = 0; ; attempt++) {
5955
+ let response;
5956
+ try {
5957
+ response = await fetch(url, {
5958
+ method: "POST",
5959
+ headers: requestHeaders(apiKey),
5960
+ signal: AbortSignal.timeout(READ_TIMEOUT_MS)
5961
+ });
5962
+ } catch (err) {
5963
+ if (err instanceof Error && err.name === "TimeoutError") {
5964
+ throw new Error(`No answer from ScrapeUnblocker within ${READ_TIMEOUT_MS / 1e3} s.`);
5965
+ }
5966
+ if (attempt >= READ_RETRIES) throw new Error(`Could not reach ScrapeUnblocker: ${errorText(err)}`);
5967
+ await backoff(attempt);
5968
+ continue;
5969
+ }
5970
+ if (RETRYABLE.has(response.status) && attempt < READ_RETRIES) {
5971
+ await response.body?.cancel();
5972
+ await backoff(attempt);
5973
+ continue;
5974
+ }
5975
+ const body = await response.text();
5976
+ if (!response.ok) throw errorForStatus(response.status, body, response.headers);
5977
+ return body;
5978
+ }
5979
+ }
6068
5980
  var server = new McpServer({
6069
5981
  name: "scrapeunblocker",
6070
5982
  version: VERSION
@@ -6137,7 +6049,7 @@ server.registerTool(
6137
6049
  {
6138
6050
  title: "Fetch page HTML",
6139
6051
  annotations: { title: "Fetch page HTML", readOnlyHint: true, openWorldHint: true, destructiveHint: false },
6140
- description: "Fetch the fully rendered HTML of any web page through ScrapeUnblocker, bypassing anti-bot protection (Cloudflare, DataDome, PerimeterX, Akamai, Shape). Use this when a normal fetch is blocked (403/429, captcha, 'access denied') or when the page needs a real browser to render. Returns the raw HTML as text.\n\nFor pages that need interaction (accept a cookie banner, click a tab, type into a search box, scroll to trigger lazy loading) pass `steps`: an ordered list of browser actions run in a real browser AFTER the page loads, then the resulting HTML is returned. Workflow: first call the `list_elements` tool to discover the real selectors on the page, then build `steps` against them. Steps are NOT idempotent - they run once per call. If a step fails, this tool returns which step failed, why, and the page HTML at that moment so you can fix the selector and retry.",
6052
+ description: "Fetch the fully rendered HTML of any web page through ScrapeUnblocker, bypassing anti-bot protection (Cloudflare, DataDome, PerimeterX, Akamai, Shape). Use this when a normal fetch is blocked (403/429, captcha, 'access denied') or when the page needs a real browser to render. Returns the raw HTML as text. To read or summarise a page, fetch_markdown is much smaller. To click, type or scroll before capturing, use browse_page.",
6141
6053
  inputSchema: {
6142
6054
  url: z3.string().url().describe("The absolute URL to fetch (http/https)."),
6143
6055
  proxy_country: z3.string().length(2).optional().describe(
@@ -6150,24 +6062,54 @@ server.registerTool(
6150
6062
  `The CSS selector or JS expression paired with wait_method (e.g. '#price' or 'document.readyState==="complete"').`
6151
6063
  ),
6152
6064
  method_timeout_seconds: z3.number().int().positive().optional().describe("Cap in seconds for the render-wait method."),
6153
- sleep_seconds: z3.number().positive().optional().describe("Extra seconds to wait after load before capturing the HTML."),
6154
- steps: z3.array(stepSchema).optional().describe(
6155
- "Optional ordered browser actions to run in a real browser after the page loads, before the HTML is captured. Use `list_elements` first to find selectors. Runs once (not idempotent)."
6156
- )
6065
+ sleep_seconds: z3.number().positive().optional().describe("Extra seconds to wait after load before capturing the HTML.")
6066
+ }
6067
+ },
6068
+ async (args) => {
6069
+ try {
6070
+ const html = await client().getPageSource(args.url, {
6071
+ proxyCountry: args.proxy_country,
6072
+ method: args.wait_method,
6073
+ value: args.wait_value,
6074
+ methodTimeout: args.method_timeout_seconds,
6075
+ timeSleep: args.sleep_seconds
6076
+ });
6077
+ return { content: [{ type: "text", text: html }] };
6078
+ } catch (err) {
6079
+ return errorResult(args.url, err);
6080
+ }
6081
+ }
6082
+ );
6083
+ server.registerTool(
6084
+ "browse_page",
6085
+ {
6086
+ title: "Interact with a page, then capture it",
6087
+ annotations: {
6088
+ title: "Interact with a page, then capture it",
6089
+ readOnlyHint: false,
6090
+ destructiveHint: true,
6091
+ idempotentHint: false,
6092
+ openWorldHint: true
6093
+ },
6094
+ description: "Load a web page in a real browser through ScrapeUnblocker, run an ordered list of browser actions on it (accept a cookie banner, click a tab, type into a search box, choose an option, press a key, scroll to trigger lazy loading, wait), then return the resulting HTML. Workflow: first call the `list_elements` tool to discover the real selectors on the page, then build `steps` against them. Steps act on the website and are NOT idempotent - they run once per call; never use them to buy, post, delete or change anything unless the user asked for exactly that. If a step fails, this tool returns which step failed, why, and the page HTML at that moment so you can fix the selector and retry. Billed as one request.",
6095
+ inputSchema: {
6096
+ url: z3.string().url().describe("The absolute URL to open (http/https)."),
6097
+ steps: z3.array(stepSchema).min(1).describe(
6098
+ "Ordered browser actions to run in a real browser after the page loads, before the HTML is captured. Use `list_elements` first to find selectors. Runs once (not idempotent)."
6099
+ ),
6100
+ proxy_country: z3.string().length(2).optional().describe(
6101
+ "Optional ISO 3166-1 alpha-2 country code to route through, e.g. 'US', 'GB', 'DE'."
6102
+ ),
6103
+ wait_method: z3.enum(["css", "js"]).optional().describe(
6104
+ "Optional render-wait before the steps run: 'css' waits for a selector, 'js' waits for a JS expression to be truthy."
6105
+ ),
6106
+ wait_value: z3.string().optional().describe("The CSS selector or JS expression paired with wait_method."),
6107
+ method_timeout_seconds: z3.number().int().positive().optional().describe("Cap in seconds for the render-wait method."),
6108
+ sleep_seconds: z3.number().positive().optional().describe("Extra seconds to wait after load.")
6157
6109
  }
6158
6110
  },
6159
6111
  async (args) => {
6160
6112
  try {
6161
- if (!args.steps || args.steps.length === 0) {
6162
- const html = await client().getPageSource(args.url, {
6163
- proxyCountry: args.proxy_country,
6164
- method: args.wait_method,
6165
- value: args.wait_value,
6166
- methodTimeout: args.method_timeout_seconds,
6167
- timeSleep: args.sleep_seconds
6168
- });
6169
- return { content: [{ type: "text", text: html }] };
6170
- }
6171
6113
  const res = await getPageSourceRaw({
6172
6114
  url: args.url,
6173
6115
  proxy_country: args.proxy_country,
@@ -6223,7 +6165,7 @@ server.registerTool(
6223
6165
  {
6224
6166
  title: "List page elements (selector discovery)",
6225
6167
  annotations: { title: "List page elements (selector discovery)", readOnlyHint: true, openWorldHint: true, destructiveHint: false },
6226
- description: "Load a page through ScrapeUnblocker and return a JSON list of its notable elements (links, inputs, buttons, selects, etc.) with a ready-to-use `selector` for each, plus tag, text and useful attributes (name, id, type, placeholder, aria_label, href, ...). Read-only and does not interact with the page. This is the discovery half of interactive scraping: call `list_elements` to find the selectors you need, then pass matching `steps` to `fetch_html` to click/type/select and capture the resulting HTML.",
6168
+ description: "Load a page through ScrapeUnblocker and return a JSON list of its notable elements (links, inputs, buttons, selects, etc.) with a ready-to-use `selector` for each, plus tag, text and useful attributes (name, id, type, placeholder, aria_label, href, ...). Read-only and does not interact with the page. This is the discovery half of interactive scraping: call `list_elements` to find the selectors you need, then pass matching `steps` to `browse_page` to click/type/select and capture the resulting HTML.",
6227
6169
  inputSchema: {
6228
6170
  url: z3.string().url().describe("The absolute URL whose elements you want to list."),
6229
6171
  proxy_country: z3.string().length(2).optional().describe("Optional ISO country code to route through, e.g. 'US'.")
@@ -6321,7 +6263,7 @@ server.registerTool(
6321
6263
  }
6322
6264
  );
6323
6265
  registerReadingTools(server, {
6324
- fetchHtml: (url, options) => client().getPageSource(url, options),
6266
+ getPageSource: readPageSource,
6325
6267
  serp: (keyword, options) => client().serp(keyword, options),
6326
6268
  fetchError: errorResult,
6327
6269
  errorText
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "scrapeunblocker-mcp",
3
- "version": "0.5.0",
3
+ "version": "0.7.0",
4
4
  "mcpName": "com.scrapeunblocker/scrapeunblocker-mcp",
5
5
  "description": "Model Context Protocol (MCP) server for ScrapeUnblocker - lets Claude fetch any web page's HTML (or AI-parsed JSON) and run 100+ site plugins that return structured JSON, through the ScrapeUnblocker anti-bot API, using your own API key.",
6
6
  "keywords": [
@@ -37,21 +37,18 @@
37
37
  "scripts": {
38
38
  "build": "tsup",
39
39
  "typecheck": "tsc --noEmit",
40
+ "test": "npm run build && node --test test/*.test.mjs",
40
41
  "start": "node dist/index.js",
41
42
  "dev": "tsx src/index.ts",
42
43
  "prepublishOnly": "npm run build"
43
44
  },
44
45
  "dependencies": {
45
46
  "@modelcontextprotocol/sdk": "^1.32.1",
46
- "linkedom": "^0.18.13",
47
47
  "scrapeunblocker": "^0.10.0",
48
- "turndown": "^7.2.4",
49
- "turndown-plugin-gfm": "^1.0.2",
50
48
  "zod": "^3.23.8"
51
49
  },
52
50
  "devDependencies": {
53
51
  "@types/node": "^20.19.43",
54
- "@types/turndown": "^5.0.6",
55
52
  "tsup": "^8.0.0",
56
53
  "tsx": "^4.16.0",
57
54
  "typescript": "^5.4.0"