zames_pro 2.53.1 → 2.54.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/web.js +101 -2
- package/package.json +1 -1
package/dist/web.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { chromium } from 'playwright';
|
|
2
|
+
import { lookup } from 'node:dns/promises';
|
|
3
|
+
import { isIP } from 'node:net';
|
|
2
4
|
const DEFAULT_TIMEOUT = 20_000;
|
|
3
5
|
const MAX_TEXT = 12_000;
|
|
4
6
|
// Removes scripts, styles, and turns HTML into readable text.
|
|
@@ -42,6 +44,102 @@ function htmlToText(html) {
|
|
|
42
44
|
.join('\n');
|
|
43
45
|
return s;
|
|
44
46
|
}
|
|
47
|
+
// ---------- SSRF guard ----------
|
|
48
|
+
//
|
|
49
|
+
// WebFetch takes an arbitrary URL from the model. Without a guard it could be
|
|
50
|
+
// talked into reading cloud metadata (169.254.169.254), internal admin panels
|
|
51
|
+
// or localhost services. We resolve the hostname and refuse private / loopback
|
|
52
|
+
// / link-local addresses. Best-effort: a DNS rebind between the check and the
|
|
53
|
+
// fetch is out of scope for a CLI tool, but the common cases are covered.
|
|
54
|
+
export function isPrivateIp(ip) {
|
|
55
|
+
const v4 = ip.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
56
|
+
if (v4) {
|
|
57
|
+
const [a, b] = v4.slice(1).map(Number);
|
|
58
|
+
if (a === 10 || a === 127 || a === 0)
|
|
59
|
+
return true;
|
|
60
|
+
if (a === 169 && b === 254)
|
|
61
|
+
return true; // link-local + cloud metadata
|
|
62
|
+
if (a === 172 && b >= 16 && b <= 31)
|
|
63
|
+
return true;
|
|
64
|
+
if (a === 192 && b === 168)
|
|
65
|
+
return true;
|
|
66
|
+
if (a === 100 && b >= 64 && b <= 127)
|
|
67
|
+
return true; // CGNAT
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
const low = ip.toLowerCase();
|
|
71
|
+
if (low === '::1' || low === '::')
|
|
72
|
+
return true;
|
|
73
|
+
if (low.startsWith('fe80') || low.startsWith('fc') || low.startsWith('fd')) {
|
|
74
|
+
return true;
|
|
75
|
+
}
|
|
76
|
+
const mapped = low.match(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/);
|
|
77
|
+
if (mapped)
|
|
78
|
+
return isPrivateIp(mapped[1]);
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
export function isPrivateHostname(host) {
|
|
82
|
+
const h = host.toLowerCase().replace(/\.$/, '');
|
|
83
|
+
if (!h)
|
|
84
|
+
return true;
|
|
85
|
+
if (h === 'localhost' || h.endsWith('.localhost'))
|
|
86
|
+
return true;
|
|
87
|
+
if (h === 'metadata.google.internal')
|
|
88
|
+
return true;
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
async function assertPublicUrl(rawUrl) {
|
|
92
|
+
const u = new URL(rawUrl);
|
|
93
|
+
if (u.protocol !== 'http:' && u.protocol !== 'https:') {
|
|
94
|
+
throw new Error(`unsupported protocol: ${u.protocol}`);
|
|
95
|
+
}
|
|
96
|
+
const host = u.hostname;
|
|
97
|
+
if (isPrivateHostname(host)) {
|
|
98
|
+
throw new Error(`blocked private host: ${host}`);
|
|
99
|
+
}
|
|
100
|
+
if (isIP(host)) {
|
|
101
|
+
if (isPrivateIp(host))
|
|
102
|
+
throw new Error(`blocked private address: ${host}`);
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
try {
|
|
106
|
+
const addrs = await lookup(host, { all: true });
|
|
107
|
+
for (const a of addrs) {
|
|
108
|
+
if (isPrivateIp(a.address)) {
|
|
109
|
+
throw new Error(`blocked private address for ${host}: ${a.address}`);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
catch (e) {
|
|
114
|
+
// Re-throw our own block, but let a DNS failure fall through so fetch()
|
|
115
|
+
// surfaces the real network error instead of a misleading one.
|
|
116
|
+
if (e.message.startsWith('blocked'))
|
|
117
|
+
throw e;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
121
|
+
// fetch with retries for transient network errors and 5xx responses.
|
|
122
|
+
async function httpFetchWithRetry(url, opts = {}, retries = 2) {
|
|
123
|
+
let lastErr;
|
|
124
|
+
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
125
|
+
try {
|
|
126
|
+
const r = await httpFetch(url, opts);
|
|
127
|
+
if (r.status >= 500 && attempt < retries) {
|
|
128
|
+
await sleep(300 * 2 ** attempt);
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
return r;
|
|
132
|
+
}
|
|
133
|
+
catch (e) {
|
|
134
|
+
lastErr = e;
|
|
135
|
+
if (attempt < retries) {
|
|
136
|
+
await sleep(300 * 2 ** attempt);
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
|
|
142
|
+
}
|
|
45
143
|
// fetch with redirects and a timeout
|
|
46
144
|
async function httpFetch(url, { timeout = DEFAULT_TIMEOUT, headers = {}, } = {}) {
|
|
47
145
|
const ctrl = new AbortController();
|
|
@@ -126,12 +224,13 @@ export function createWebTools() {
|
|
|
126
224
|
}
|
|
127
225
|
const limit = Math.min(Number(maxChars) || MAX_TEXT, 60_000);
|
|
128
226
|
try {
|
|
227
|
+
await assertPublicUrl(String(url));
|
|
129
228
|
if (render) {
|
|
130
229
|
const r = await renderWithHeadless(String(url));
|
|
131
230
|
const text = htmlToText(r.html);
|
|
132
231
|
return formatResult(r.status, r.url, text, limit);
|
|
133
232
|
}
|
|
134
|
-
const r = await
|
|
233
|
+
const r = await httpFetchWithRetry(String(url));
|
|
135
234
|
// If it's JSON/plain text — return as is
|
|
136
235
|
if (/application\/json|text\/plain|text\/markdown/i.test(r.contentType)) {
|
|
137
236
|
const trimmed = r.body.slice(0, limit);
|
|
@@ -168,7 +267,7 @@ export function createWebTools() {
|
|
|
168
267
|
return 'Error: query is empty.';
|
|
169
268
|
const limit = Math.min(Math.max(Number(maxResults) || 8, 1), 20);
|
|
170
269
|
try {
|
|
171
|
-
const r = await
|
|
270
|
+
const r = await httpFetchWithRetry(`https://html.duckduckgo.com/html/?q=${q}`, {
|
|
172
271
|
timeout: 25_000,
|
|
173
272
|
headers: {
|
|
174
273
|
Accept: 'text/html,application/xhtml+xml',
|