scrapeunblocker-mcp 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -6
- package/dist/index.js +135 -193
- package/package.json +2 -5
package/README.md
CHANGED
|
@@ -12,7 +12,8 @@ You bring your **own** API key. Nothing is shared or proxied through us.
|
|
|
12
12
|
|
|
13
13
|
| Tool | What it does |
|
|
14
14
|
|------|--------------|
|
|
15
|
-
| `fetch_html` | Fetch the fully rendered HTML of a URL
|
|
15
|
+
| `fetch_html` | Fetch the fully rendered HTML of a URL. |
|
|
16
|
+
| `browse_page` | Run interactive browser `steps` on a page (click, type, select, scroll, wait), then return the resulting HTML. Not read-only: it acts on the website. |
|
|
16
17
|
| `list_elements` | List a page's notable elements with a ready-to-use selector for each (selector discovery). |
|
|
17
18
|
| `fetch_parsed` | Fetch a page and return AI-parsed structured JSON. |
|
|
18
19
|
| `google_search` | Run a Google search and return organic results as JSON. |
|
|
@@ -43,7 +44,7 @@ Each run is billed like an API request.
|
|
|
43
44
|
|
|
44
45
|
Some pages only reveal what you need after you interact with them - accept a
|
|
45
46
|
cookie banner, click a tab, type into a search box and submit, or scroll to
|
|
46
|
-
trigger lazy loading. Pass
|
|
47
|
+
trigger lazy loading. Pass a `steps` array to `browse_page` and those
|
|
47
48
|
actions run in a **real browser**, in order, after the page loads; the resulting
|
|
48
49
|
HTML is then returned.
|
|
49
50
|
|
|
@@ -60,8 +61,9 @@ Available actions:
|
|
|
60
61
|
| `press_key` | `value` (`Enter`, `Tab`, `Escape`, `Backspace`, `Delete`, `Space`, `Arrow*`, `Home`, `End`, `PageUp`, `PageDown`) |
|
|
61
62
|
| `scroll` | `value` (`"bottom"` or a pixel offset) |
|
|
62
63
|
|
|
63
|
-
Steps are **not idempotent** - they run once per call
|
|
64
|
-
`
|
|
64
|
+
Steps are **not idempotent** - they run once per call - and they act on the
|
|
65
|
+
website, so `browse_page` is the one tool not marked read-only. If a step fails,
|
|
66
|
+
`browse_page` returns which step failed, why, and the page HTML at that moment so
|
|
65
67
|
you can fix the selector and retry.
|
|
66
68
|
|
|
67
69
|
### List elements (discover selectors first)
|
|
@@ -76,7 +78,7 @@ you can fix the selector and retry.
|
|
|
76
78
|
```
|
|
77
79
|
|
|
78
80
|
The natural workflow is **discover, then act**: call `list_elements` to find the
|
|
79
|
-
selectors you need, then pass matching `steps` to `
|
|
81
|
+
selectors you need, then pass matching `steps` to `browse_page` to click/type/
|
|
80
82
|
select and capture the resulting HTML.
|
|
81
83
|
|
|
82
84
|
## Get an API key
|
|
@@ -140,7 +142,7 @@ SCRAPEUNBLOCKER_KEY=your_api_key_here npx -y scrapeunblocker-mcp
|
|
|
140
142
|
|
|
141
143
|
- "Fetch the HTML of https://www.example-shop.com/product/123 and list the price."
|
|
142
144
|
- "This page keeps blocking me: <url>. Use fetch_html to get it."
|
|
143
|
-
- "List the elements on <url>, then use
|
|
145
|
+
- "List the elements on <url>, then use browse_page to type 'laptop' into the search box, press Enter, wait for the results, and give me the HTML."
|
|
144
146
|
- "Search Google for 'best running shoes 2026' and give me the top 5 links."
|
|
145
147
|
- "Read https://en.wikipedia.org/wiki/Web_scraping as Markdown and summarise it in five bullets."
|
|
146
148
|
- "Search the web for the current population of Vilnius and cite the pages you read."
|
|
@@ -154,6 +156,7 @@ SCRAPEUNBLOCKER_KEY=your_api_key_here npx -y scrapeunblocker-mcp
|
|
|
154
156
|
npm install
|
|
155
157
|
npm run build # bundles to dist/ with tsup
|
|
156
158
|
npm run typecheck
|
|
159
|
+
npm test # builds, then runs the tools against a local stand-in API
|
|
157
160
|
SCRAPEUNBLOCKER_KEY=... node dist/index.js # run the server
|
|
158
161
|
```
|
|
159
162
|
|
package/dist/index.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
5
5
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
6
6
|
import { z as z3 } from "zod";
|
|
7
|
-
import { APIError, ScrapeUnblockerClient } from "scrapeunblocker";
|
|
7
|
+
import { APIError, ScrapeUnblockerClient, errorForStatus } from "scrapeunblocker";
|
|
8
8
|
|
|
9
9
|
// src/plugin-tools.ts
|
|
10
10
|
import { z } from "zod";
|
|
@@ -5688,155 +5688,6 @@ ${details(plugin)}`, true);
|
|
|
5688
5688
|
|
|
5689
5689
|
// src/reading-tools.ts
|
|
5690
5690
|
import { z as z2 } from "zod";
|
|
5691
|
-
|
|
5692
|
-
// src/markdown.ts
|
|
5693
|
-
import { parseHTML } from "linkedom";
|
|
5694
|
-
import TurndownService from "turndown";
|
|
5695
|
-
import { gfm } from "turndown-plugin-gfm";
|
|
5696
|
-
var ALWAYS_DROP = [
|
|
5697
|
-
"script",
|
|
5698
|
-
"style",
|
|
5699
|
-
"noscript",
|
|
5700
|
-
"template",
|
|
5701
|
-
"svg",
|
|
5702
|
-
"canvas",
|
|
5703
|
-
"iframe",
|
|
5704
|
-
"object",
|
|
5705
|
-
"embed",
|
|
5706
|
-
"link",
|
|
5707
|
-
"meta",
|
|
5708
|
-
"head",
|
|
5709
|
-
"button",
|
|
5710
|
-
"input",
|
|
5711
|
-
"select",
|
|
5712
|
-
"textarea",
|
|
5713
|
-
"dialog",
|
|
5714
|
-
"[hidden]",
|
|
5715
|
-
'[aria-hidden="true"]'
|
|
5716
|
-
].join(",");
|
|
5717
|
-
var CHROME = ["nav", "footer", "aside", '[role="navigation"]', '[role="contentinfo"]', '[role="complementary"]'].join(",");
|
|
5718
|
-
var MAIN_MIN_SHARE = 0.3;
|
|
5719
|
-
function textLength(el) {
|
|
5720
|
-
return (el?.textContent ?? "").replace(/\s+/g, " ").trim().length;
|
|
5721
|
-
}
|
|
5722
|
-
function absolute(href, base) {
|
|
5723
|
-
try {
|
|
5724
|
-
const url = new URL(href, base);
|
|
5725
|
-
return url.protocol === "http:" || url.protocol === "https:" ? url.href : null;
|
|
5726
|
-
} catch {
|
|
5727
|
-
return null;
|
|
5728
|
-
}
|
|
5729
|
-
}
|
|
5730
|
-
function converter() {
|
|
5731
|
-
const td = new TurndownService({
|
|
5732
|
-
headingStyle: "atx",
|
|
5733
|
-
codeBlockStyle: "fenced",
|
|
5734
|
-
bulletListMarker: "-",
|
|
5735
|
-
emDelimiter: "_"
|
|
5736
|
-
});
|
|
5737
|
-
td.use(gfm);
|
|
5738
|
-
td.addRule("images", {
|
|
5739
|
-
filter: "img",
|
|
5740
|
-
replacement: (_content, node) => {
|
|
5741
|
-
const el = node;
|
|
5742
|
-
const alt = (el.getAttribute("alt") || "").replace(/\s+/g, " ").trim();
|
|
5743
|
-
const src = el.getAttribute("src") || "";
|
|
5744
|
-
return alt && /^https?:/i.test(src) ? `` : "";
|
|
5745
|
-
}
|
|
5746
|
-
});
|
|
5747
|
-
td.addRule("flatCells", {
|
|
5748
|
-
filter: ["th", "td"],
|
|
5749
|
-
replacement: (content, node) => {
|
|
5750
|
-
const cell = node;
|
|
5751
|
-
const first = Array.prototype.indexOf.call(cell.parentNode.childNodes, cell) === 0;
|
|
5752
|
-
const text3 = content.replace(/\s*\n+\s*/g, " ").replace(/\|/g, "\\|").trim();
|
|
5753
|
-
return `${first ? "| " : " "}${text3} |`;
|
|
5754
|
-
}
|
|
5755
|
-
});
|
|
5756
|
-
return td;
|
|
5757
|
-
}
|
|
5758
|
-
function normalizeTables(document) {
|
|
5759
|
-
for (const table of [...document.querySelectorAll("table")].reverse()) {
|
|
5760
|
-
const rows = [...table.querySelectorAll("tr")].filter((tr) => tr.closest("table") === table).map((tr) => [...tr.children].filter((c) => c.tagName === "TD" || c.tagName === "TH")).filter((cells) => cells.length > 0);
|
|
5761
|
-
if (rows.length === 0) {
|
|
5762
|
-
table.remove();
|
|
5763
|
-
continue;
|
|
5764
|
-
}
|
|
5765
|
-
if (rows.length === 1 || Math.max(...rows.map((cells) => cells.length)) === 1) {
|
|
5766
|
-
const blocks = document.createElement("div");
|
|
5767
|
-
for (const [cell] of rows) {
|
|
5768
|
-
const block = document.createElement("div");
|
|
5769
|
-
block.innerHTML = cell.innerHTML;
|
|
5770
|
-
blocks.appendChild(block);
|
|
5771
|
-
}
|
|
5772
|
-
table.replaceWith(blocks);
|
|
5773
|
-
continue;
|
|
5774
|
-
}
|
|
5775
|
-
const clean = document.createElement("table");
|
|
5776
|
-
rows.forEach((cells, i) => {
|
|
5777
|
-
const section = document.createElement(i === 0 ? "thead" : "tbody");
|
|
5778
|
-
const tr = document.createElement("tr");
|
|
5779
|
-
for (const cell of cells) {
|
|
5780
|
-
const out = document.createElement(i === 0 ? "th" : "td");
|
|
5781
|
-
out.innerHTML = cell.innerHTML.replace(/\s+/g, " ");
|
|
5782
|
-
tr.appendChild(out);
|
|
5783
|
-
}
|
|
5784
|
-
if (i === 0 || !clean.lastElementChild || clean.lastElementChild.tagName !== "TBODY") {
|
|
5785
|
-
section.appendChild(tr);
|
|
5786
|
-
clean.appendChild(section);
|
|
5787
|
-
} else {
|
|
5788
|
-
clean.lastElementChild.appendChild(tr);
|
|
5789
|
-
}
|
|
5790
|
-
});
|
|
5791
|
-
table.replaceWith(clean);
|
|
5792
|
-
}
|
|
5793
|
-
}
|
|
5794
|
-
function htmlToMarkdown(html, url, options) {
|
|
5795
|
-
if (!/<[a-z!][\s\S]*>/i.test(html)) return { title: "", markdown: html.trim() };
|
|
5796
|
-
const { document } = parseHTML(html);
|
|
5797
|
-
const title = (document.querySelector("title")?.textContent ?? "").replace(/\s+/g, " ").trim();
|
|
5798
|
-
for (const el of document.querySelectorAll(ALWAYS_DROP)) el.remove();
|
|
5799
|
-
for (const a of document.querySelectorAll("a[href]")) {
|
|
5800
|
-
const href = absolute(a.getAttribute("href") || "", url);
|
|
5801
|
-
if (href) a.setAttribute("href", href);
|
|
5802
|
-
else a.removeAttribute("href");
|
|
5803
|
-
}
|
|
5804
|
-
for (const img of document.querySelectorAll("img")) {
|
|
5805
|
-
const src = absolute(img.getAttribute("src") || "", url);
|
|
5806
|
-
if (src) img.setAttribute("src", src);
|
|
5807
|
-
else img.removeAttribute("src");
|
|
5808
|
-
}
|
|
5809
|
-
const body = document.body ?? document.documentElement;
|
|
5810
|
-
let root = body;
|
|
5811
|
-
if (options.mainOnly) {
|
|
5812
|
-
const main2 = document.querySelector('main, [role="main"]');
|
|
5813
|
-
if (main2 && textLength(main2) >= MAIN_MIN_SHARE * textLength(body)) {
|
|
5814
|
-
root = main2;
|
|
5815
|
-
} else {
|
|
5816
|
-
for (const el of document.querySelectorAll(CHROME)) el.remove();
|
|
5817
|
-
for (const header of document.querySelectorAll('header, [role="banner"]')) {
|
|
5818
|
-
if (!header.closest("article, main")) header.remove();
|
|
5819
|
-
}
|
|
5820
|
-
}
|
|
5821
|
-
}
|
|
5822
|
-
normalizeTables(document);
|
|
5823
|
-
let converted;
|
|
5824
|
-
try {
|
|
5825
|
-
converted = converter().turndown(root.innerHTML);
|
|
5826
|
-
} catch {
|
|
5827
|
-
converted = (root.textContent ?? "").replace(/[ \t]+/g, " ").replace(/\s*\n\s*/g, "\n");
|
|
5828
|
-
}
|
|
5829
|
-
const markdown = converted.replace(/\[\s*\]\([^)]*\)/g, "").replace(/^(\s*)([-*+]|\d+\.) {2,}/gm, "$1$2 ").replace(/(\)|\*\*|_) ([.,;:!?])/g, "$1$2").replace(/[ \t]+$/gm, "").replace(/\n{3,}/g, "\n\n").trim();
|
|
5830
|
-
return { title, markdown };
|
|
5831
|
-
}
|
|
5832
|
-
function capText(text3, max, hint) {
|
|
5833
|
-
if (text3.length <= max) return text3;
|
|
5834
|
-
return `${text3.slice(0, max)}
|
|
5835
|
-
|
|
5836
|
-
[Cut at ${max} of ${text3.length} characters. ${hint}]`;
|
|
5837
|
-
}
|
|
5838
|
-
|
|
5839
|
-
// src/reading-tools.ts
|
|
5840
5691
|
var PAGE_MAX_CHARS = 1e5;
|
|
5841
5692
|
var SEARCH_PAGE_MAX_CHARS = 15e3;
|
|
5842
5693
|
var SEARCH_DEFAULT_RESULTS = 3;
|
|
@@ -5845,13 +5696,27 @@ var SEARCH_PAGE_DEADLINE_MS = 12e4;
|
|
|
5845
5696
|
function text2(body, isError = false) {
|
|
5846
5697
|
return isError ? { content: [{ type: "text", text: body }], isError } : { content: [{ type: "text", text: body }] };
|
|
5847
5698
|
}
|
|
5848
|
-
function
|
|
5849
|
-
|
|
5850
|
-
|
|
5851
|
-
|
|
5852
|
-
|
|
5699
|
+
function markdownParams(url, options) {
|
|
5700
|
+
return {
|
|
5701
|
+
url,
|
|
5702
|
+
markdown: true,
|
|
5703
|
+
main_content_only: options.mainContentOnly,
|
|
5704
|
+
proxy_country: options.proxyCountry,
|
|
5705
|
+
method: options.method,
|
|
5706
|
+
value: options.value,
|
|
5707
|
+
time_sleep: options.timeSleep
|
|
5708
|
+
};
|
|
5709
|
+
}
|
|
5710
|
+
function capText(text3, max, hint) {
|
|
5711
|
+
if (text3.length <= max) return text3;
|
|
5712
|
+
return `${text3.slice(0, max)}
|
|
5713
|
+
|
|
5714
|
+
[Cut at ${max} of ${text3.length} characters. ${hint}]`;
|
|
5715
|
+
}
|
|
5716
|
+
function pageText(url, markdown, max) {
|
|
5717
|
+
return capText(`URL: ${url}
|
|
5853
5718
|
|
|
5854
|
-
${
|
|
5719
|
+
${markdown.trim()}`, max, "Use fetch_html for the whole page.");
|
|
5855
5720
|
}
|
|
5856
5721
|
function organicResults(serp) {
|
|
5857
5722
|
const organic = serp?.organic;
|
|
@@ -5907,13 +5772,16 @@ function registerReadingTools(server2, options) {
|
|
|
5907
5772
|
const blocked = options.blocked?.();
|
|
5908
5773
|
if (blocked) return blocked;
|
|
5909
5774
|
try {
|
|
5910
|
-
const
|
|
5911
|
-
|
|
5912
|
-
|
|
5913
|
-
|
|
5914
|
-
|
|
5915
|
-
|
|
5916
|
-
|
|
5775
|
+
const markdown = await options.getPageSource(
|
|
5776
|
+
markdownParams(args.url, {
|
|
5777
|
+
mainContentOnly: args.main_content_only ?? true,
|
|
5778
|
+
proxyCountry: args.proxy_country,
|
|
5779
|
+
method: args.wait_method,
|
|
5780
|
+
value: args.wait_value,
|
|
5781
|
+
timeSleep: args.sleep_seconds
|
|
5782
|
+
})
|
|
5783
|
+
);
|
|
5784
|
+
return text2(pageText(args.url, markdown, PAGE_MAX_CHARS));
|
|
5917
5785
|
} catch (err) {
|
|
5918
5786
|
return options.fetchError(args.url, err);
|
|
5919
5787
|
}
|
|
@@ -5948,7 +5816,10 @@ function registerReadingTools(server2, options) {
|
|
|
5948
5816
|
const toRead = organic.slice(0, args.results ?? SEARCH_DEFAULT_RESULTS);
|
|
5949
5817
|
const pages = await Promise.allSettled(
|
|
5950
5818
|
toRead.map(
|
|
5951
|
-
(r) => withDeadline(
|
|
5819
|
+
(r) => withDeadline(
|
|
5820
|
+
options.getPageSource(markdownParams(r.url, { mainContentOnly: true, proxyCountry: args.proxy_country })),
|
|
5821
|
+
SEARCH_PAGE_DEADLINE_MS
|
|
5822
|
+
)
|
|
5952
5823
|
)
|
|
5953
5824
|
);
|
|
5954
5825
|
const parts = [
|
|
@@ -5959,7 +5830,7 @@ function registerReadingTools(server2, options) {
|
|
|
5959
5830
|
${overview}`);
|
|
5960
5831
|
toRead.forEach((r, i) => {
|
|
5961
5832
|
const page = pages[i];
|
|
5962
|
-
const body = page.status === "fulfilled" ?
|
|
5833
|
+
const body = page.status === "fulfilled" ? pageText(r.url, page.value, SEARCH_PAGE_MAX_CHARS) : `URL: ${r.url}
|
|
5963
5834
|
|
|
5964
5835
|
Could not read this page: ${options.errorText(page.reason)}
|
|
5965
5836
|
Snippet: ${r.description}`;
|
|
@@ -5979,7 +5850,7 @@ ${body}`);
|
|
|
5979
5850
|
}
|
|
5980
5851
|
|
|
5981
5852
|
// src/index.ts
|
|
5982
|
-
var VERSION = "0.
|
|
5853
|
+
var VERSION = "0.7.0";
|
|
5983
5854
|
var DEFAULT_BASE_URL = "https://api.scrapeunblocker.com";
|
|
5984
5855
|
var API_KEY_HEADER = "x-scrapeunblocker-key";
|
|
5985
5856
|
function resolveApiKey() {
|
|
@@ -6040,22 +5911,27 @@ function errorResult(url, err) {
|
|
|
6040
5911
|
}
|
|
6041
5912
|
return { content: [{ type: "text", text: errorText(err) }], isError: true };
|
|
6042
5913
|
}
|
|
6043
|
-
|
|
6044
|
-
const apiKey = resolveApiKey();
|
|
6045
|
-
if (!apiKey) throw new Error(MISSING_KEY_MESSAGE);
|
|
5914
|
+
function pageSourceUrl(params) {
|
|
6046
5915
|
const query = new URLSearchParams();
|
|
6047
5916
|
for (const [key, value] of Object.entries(params)) {
|
|
6048
5917
|
if (value === void 0 || value === null) continue;
|
|
6049
5918
|
query.set(key, String(value));
|
|
6050
5919
|
}
|
|
6051
|
-
|
|
6052
|
-
|
|
5920
|
+
return `${resolveBaseUrl()}/getPageSource?${query.toString()}`;
|
|
5921
|
+
}
|
|
5922
|
+
function requestHeaders(apiKey) {
|
|
5923
|
+
return {
|
|
5924
|
+
[API_KEY_HEADER]: apiKey,
|
|
5925
|
+
"User-Agent": `scrapeunblocker-mcp/${VERSION}`,
|
|
5926
|
+
Accept: "*/*"
|
|
5927
|
+
};
|
|
5928
|
+
}
|
|
5929
|
+
async function getPageSourceRaw(params) {
|
|
5930
|
+
const apiKey = resolveApiKey();
|
|
5931
|
+
if (!apiKey) throw new Error(MISSING_KEY_MESSAGE);
|
|
5932
|
+
const response = await fetch(pageSourceUrl(params), {
|
|
6053
5933
|
method: "POST",
|
|
6054
|
-
headers:
|
|
6055
|
-
[API_KEY_HEADER]: apiKey,
|
|
6056
|
-
"User-Agent": `scrapeunblocker-mcp/${VERSION}`,
|
|
6057
|
-
Accept: "*/*"
|
|
6058
|
-
}
|
|
5934
|
+
headers: requestHeaders(apiKey)
|
|
6059
5935
|
});
|
|
6060
5936
|
const text3 = await response.text();
|
|
6061
5937
|
return {
|
|
@@ -6065,6 +5941,42 @@ async function getPageSourceRaw(params) {
|
|
|
6065
5941
|
originStatus: response.headers.get("x-origin-status")
|
|
6066
5942
|
};
|
|
6067
5943
|
}
|
|
5944
|
+
var READ_TIMEOUT_MS = 18e4;
|
|
5945
|
+
var READ_RETRIES = 2;
|
|
5946
|
+
var RETRYABLE = /* @__PURE__ */ new Set([429, 502, 503, 504]);
|
|
5947
|
+
function backoff(attempt) {
|
|
5948
|
+
return new Promise((resolve) => setTimeout(resolve, Math.min(500 * 2 ** attempt, 8e3)));
|
|
5949
|
+
}
|
|
5950
|
+
async function readPageSource(params) {
|
|
5951
|
+
const apiKey = resolveApiKey();
|
|
5952
|
+
if (!apiKey) throw new Error(MISSING_KEY_MESSAGE);
|
|
5953
|
+
const url = pageSourceUrl(params);
|
|
5954
|
+
for (let attempt = 0; ; attempt++) {
|
|
5955
|
+
let response;
|
|
5956
|
+
try {
|
|
5957
|
+
response = await fetch(url, {
|
|
5958
|
+
method: "POST",
|
|
5959
|
+
headers: requestHeaders(apiKey),
|
|
5960
|
+
signal: AbortSignal.timeout(READ_TIMEOUT_MS)
|
|
5961
|
+
});
|
|
5962
|
+
} catch (err) {
|
|
5963
|
+
if (err instanceof Error && err.name === "TimeoutError") {
|
|
5964
|
+
throw new Error(`No answer from ScrapeUnblocker within ${READ_TIMEOUT_MS / 1e3} s.`);
|
|
5965
|
+
}
|
|
5966
|
+
if (attempt >= READ_RETRIES) throw new Error(`Could not reach ScrapeUnblocker: ${errorText(err)}`);
|
|
5967
|
+
await backoff(attempt);
|
|
5968
|
+
continue;
|
|
5969
|
+
}
|
|
5970
|
+
if (RETRYABLE.has(response.status) && attempt < READ_RETRIES) {
|
|
5971
|
+
await response.body?.cancel();
|
|
5972
|
+
await backoff(attempt);
|
|
5973
|
+
continue;
|
|
5974
|
+
}
|
|
5975
|
+
const body = await response.text();
|
|
5976
|
+
if (!response.ok) throw errorForStatus(response.status, body, response.headers);
|
|
5977
|
+
return body;
|
|
5978
|
+
}
|
|
5979
|
+
}
|
|
6068
5980
|
var server = new McpServer({
|
|
6069
5981
|
name: "scrapeunblocker",
|
|
6070
5982
|
version: VERSION
|
|
@@ -6137,7 +6049,7 @@ server.registerTool(
|
|
|
6137
6049
|
{
|
|
6138
6050
|
title: "Fetch page HTML",
|
|
6139
6051
|
annotations: { title: "Fetch page HTML", readOnlyHint: true, openWorldHint: true, destructiveHint: false },
|
|
6140
|
-
description: "Fetch the fully rendered HTML of any web page through ScrapeUnblocker, bypassing anti-bot protection (Cloudflare, DataDome, PerimeterX, Akamai, Shape). Use this when a normal fetch is blocked (403/429, captcha, 'access denied') or when the page needs a real browser to render. Returns the raw HTML as text
|
|
6052
|
+
description: "Fetch the fully rendered HTML of any web page through ScrapeUnblocker, bypassing anti-bot protection (Cloudflare, DataDome, PerimeterX, Akamai, Shape). Use this when a normal fetch is blocked (403/429, captcha, 'access denied') or when the page needs a real browser to render. Returns the raw HTML as text. To read or summarise a page, fetch_markdown is much smaller. To click, type or scroll before capturing, use browse_page.",
|
|
6141
6053
|
inputSchema: {
|
|
6142
6054
|
url: z3.string().url().describe("The absolute URL to fetch (http/https)."),
|
|
6143
6055
|
proxy_country: z3.string().length(2).optional().describe(
|
|
@@ -6150,24 +6062,54 @@ server.registerTool(
|
|
|
6150
6062
|
`The CSS selector or JS expression paired with wait_method (e.g. '#price' or 'document.readyState==="complete"').`
|
|
6151
6063
|
),
|
|
6152
6064
|
method_timeout_seconds: z3.number().int().positive().optional().describe("Cap in seconds for the render-wait method."),
|
|
6153
|
-
sleep_seconds: z3.number().positive().optional().describe("Extra seconds to wait after load before capturing the HTML.")
|
|
6154
|
-
|
|
6155
|
-
|
|
6156
|
-
|
|
6065
|
+
sleep_seconds: z3.number().positive().optional().describe("Extra seconds to wait after load before capturing the HTML.")
|
|
6066
|
+
}
|
|
6067
|
+
},
|
|
6068
|
+
async (args) => {
|
|
6069
|
+
try {
|
|
6070
|
+
const html = await client().getPageSource(args.url, {
|
|
6071
|
+
proxyCountry: args.proxy_country,
|
|
6072
|
+
method: args.wait_method,
|
|
6073
|
+
value: args.wait_value,
|
|
6074
|
+
methodTimeout: args.method_timeout_seconds,
|
|
6075
|
+
timeSleep: args.sleep_seconds
|
|
6076
|
+
});
|
|
6077
|
+
return { content: [{ type: "text", text: html }] };
|
|
6078
|
+
} catch (err) {
|
|
6079
|
+
return errorResult(args.url, err);
|
|
6080
|
+
}
|
|
6081
|
+
}
|
|
6082
|
+
);
|
|
6083
|
+
server.registerTool(
|
|
6084
|
+
"browse_page",
|
|
6085
|
+
{
|
|
6086
|
+
title: "Interact with a page, then capture it",
|
|
6087
|
+
annotations: {
|
|
6088
|
+
title: "Interact with a page, then capture it",
|
|
6089
|
+
readOnlyHint: false,
|
|
6090
|
+
destructiveHint: true,
|
|
6091
|
+
idempotentHint: false,
|
|
6092
|
+
openWorldHint: true
|
|
6093
|
+
},
|
|
6094
|
+
description: "Load a web page in a real browser through ScrapeUnblocker, run an ordered list of browser actions on it (accept a cookie banner, click a tab, type into a search box, choose an option, press a key, scroll to trigger lazy loading, wait), then return the resulting HTML. Workflow: first call the `list_elements` tool to discover the real selectors on the page, then build `steps` against them. Steps act on the website and are NOT idempotent - they run once per call; never use them to buy, post, delete or change anything unless the user asked for exactly that. If a step fails, this tool returns which step failed, why, and the page HTML at that moment so you can fix the selector and retry. Billed as one request.",
|
|
6095
|
+
inputSchema: {
|
|
6096
|
+
url: z3.string().url().describe("The absolute URL to open (http/https)."),
|
|
6097
|
+
steps: z3.array(stepSchema).min(1).describe(
|
|
6098
|
+
"Ordered browser actions to run in a real browser after the page loads, before the HTML is captured. Use `list_elements` first to find selectors. Runs once (not idempotent)."
|
|
6099
|
+
),
|
|
6100
|
+
proxy_country: z3.string().length(2).optional().describe(
|
|
6101
|
+
"Optional ISO 3166-1 alpha-2 country code to route through, e.g. 'US', 'GB', 'DE'."
|
|
6102
|
+
),
|
|
6103
|
+
wait_method: z3.enum(["css", "js"]).optional().describe(
|
|
6104
|
+
"Optional render-wait before the steps run: 'css' waits for a selector, 'js' waits for a JS expression to be truthy."
|
|
6105
|
+
),
|
|
6106
|
+
wait_value: z3.string().optional().describe("The CSS selector or JS expression paired with wait_method."),
|
|
6107
|
+
method_timeout_seconds: z3.number().int().positive().optional().describe("Cap in seconds for the render-wait method."),
|
|
6108
|
+
sleep_seconds: z3.number().positive().optional().describe("Extra seconds to wait after load.")
|
|
6157
6109
|
}
|
|
6158
6110
|
},
|
|
6159
6111
|
async (args) => {
|
|
6160
6112
|
try {
|
|
6161
|
-
if (!args.steps || args.steps.length === 0) {
|
|
6162
|
-
const html = await client().getPageSource(args.url, {
|
|
6163
|
-
proxyCountry: args.proxy_country,
|
|
6164
|
-
method: args.wait_method,
|
|
6165
|
-
value: args.wait_value,
|
|
6166
|
-
methodTimeout: args.method_timeout_seconds,
|
|
6167
|
-
timeSleep: args.sleep_seconds
|
|
6168
|
-
});
|
|
6169
|
-
return { content: [{ type: "text", text: html }] };
|
|
6170
|
-
}
|
|
6171
6113
|
const res = await getPageSourceRaw({
|
|
6172
6114
|
url: args.url,
|
|
6173
6115
|
proxy_country: args.proxy_country,
|
|
@@ -6223,7 +6165,7 @@ server.registerTool(
|
|
|
6223
6165
|
{
|
|
6224
6166
|
title: "List page elements (selector discovery)",
|
|
6225
6167
|
annotations: { title: "List page elements (selector discovery)", readOnlyHint: true, openWorldHint: true, destructiveHint: false },
|
|
6226
|
-
description: "Load a page through ScrapeUnblocker and return a JSON list of its notable elements (links, inputs, buttons, selects, etc.) with a ready-to-use `selector` for each, plus tag, text and useful attributes (name, id, type, placeholder, aria_label, href, ...). Read-only and does not interact with the page. This is the discovery half of interactive scraping: call `list_elements` to find the selectors you need, then pass matching `steps` to `
|
|
6168
|
+
description: "Load a page through ScrapeUnblocker and return a JSON list of its notable elements (links, inputs, buttons, selects, etc.) with a ready-to-use `selector` for each, plus tag, text and useful attributes (name, id, type, placeholder, aria_label, href, ...). Read-only and does not interact with the page. This is the discovery half of interactive scraping: call `list_elements` to find the selectors you need, then pass matching `steps` to `browse_page` to click/type/select and capture the resulting HTML.",
|
|
6227
6169
|
inputSchema: {
|
|
6228
6170
|
url: z3.string().url().describe("The absolute URL whose elements you want to list."),
|
|
6229
6171
|
proxy_country: z3.string().length(2).optional().describe("Optional ISO country code to route through, e.g. 'US'.")
|
|
@@ -6321,7 +6263,7 @@ server.registerTool(
|
|
|
6321
6263
|
}
|
|
6322
6264
|
);
|
|
6323
6265
|
registerReadingTools(server, {
|
|
6324
|
-
|
|
6266
|
+
getPageSource: readPageSource,
|
|
6325
6267
|
serp: (keyword, options) => client().serp(keyword, options),
|
|
6326
6268
|
fetchError: errorResult,
|
|
6327
6269
|
errorText
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scrapeunblocker-mcp",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"mcpName": "com.scrapeunblocker/scrapeunblocker-mcp",
|
|
5
5
|
"description": "Model Context Protocol (MCP) server for ScrapeUnblocker - lets Claude fetch any web page's HTML (or AI-parsed JSON) and run 100+ site plugins that return structured JSON, through the ScrapeUnblocker anti-bot API, using your own API key.",
|
|
6
6
|
"keywords": [
|
|
@@ -37,21 +37,18 @@
|
|
|
37
37
|
"scripts": {
|
|
38
38
|
"build": "tsup",
|
|
39
39
|
"typecheck": "tsc --noEmit",
|
|
40
|
+
"test": "npm run build && node --test test/*.test.mjs",
|
|
40
41
|
"start": "node dist/index.js",
|
|
41
42
|
"dev": "tsx src/index.ts",
|
|
42
43
|
"prepublishOnly": "npm run build"
|
|
43
44
|
},
|
|
44
45
|
"dependencies": {
|
|
45
46
|
"@modelcontextprotocol/sdk": "^1.32.1",
|
|
46
|
-
"linkedom": "^0.18.13",
|
|
47
47
|
"scrapeunblocker": "^0.10.0",
|
|
48
|
-
"turndown": "^7.2.4",
|
|
49
|
-
"turndown-plugin-gfm": "^1.0.2",
|
|
50
48
|
"zod": "^3.23.8"
|
|
51
49
|
},
|
|
52
50
|
"devDependencies": {
|
|
53
51
|
"@types/node": "^20.19.43",
|
|
54
|
-
"@types/turndown": "^5.0.6",
|
|
55
52
|
"tsup": "^8.0.0",
|
|
56
53
|
"tsx": "^4.16.0",
|
|
57
54
|
"typescript": "^5.4.0"
|