webpeel 0.20.19 → 0.20.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
const INJECTION_PATTERNS = [
|
|
9
9
|
// Direct instruction overrides
|
|
10
10
|
{ pattern: /ignore\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|rules?|prompts?|guidelines?)/gi, name: 'instruction-override' },
|
|
11
|
+
{ pattern: /ignore\s+rules?/gi, name: 'instruction-override' },
|
|
12
|
+
{ pattern: /override\s+rules?/gi, name: 'instruction-override' },
|
|
11
13
|
{ pattern: /disregard\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|rules?|prompts?)/gi, name: 'disregard-instructions' },
|
|
12
14
|
{ pattern: /forget\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|rules?|prompts?)/gi, name: 'forget-instructions' },
|
|
13
15
|
{ pattern: /override\s+(system|previous|all)\s+(prompt|instructions?|rules?)/gi, name: 'override-system' },
|
package/dist/core/youtube.js
CHANGED
|
@@ -15,11 +15,14 @@ import { join } from 'node:path';
|
|
|
15
15
|
import { fetchTranscript as ytpFetchTranscript } from 'youtube-transcript-plus';
|
|
16
16
|
import { simpleFetch } from './fetcher.js';
|
|
17
17
|
import { getBrowser, getRandomUserAgent, applyStealthScripts } from './browser-pool.js';
|
|
18
|
+
import { createLogger } from './logger.js';
|
|
18
19
|
// ---------------------------------------------------------------------------
|
|
19
20
|
// yt-dlp startup diagnostics
|
|
20
21
|
// ---------------------------------------------------------------------------
|
|
22
|
+
const _ytLog = createLogger('youtube');
|
|
21
23
|
// Check yt-dlp availability on startup.
|
|
22
24
|
// Skipped in test environments (VITEST) to avoid interfering with mocked paths.
|
|
25
|
+
// Uses logger.debug (→ stderr) so it never pollutes stdout JSON output when piped.
|
|
23
26
|
let ytdlpAvailable = false;
|
|
24
27
|
(async () => {
|
|
25
28
|
if (process.env.VITEST)
|
|
@@ -31,10 +34,10 @@ let ytdlpAvailable = false;
|
|
|
31
34
|
env: { ...process.env, PATH: `/usr/local/bin:/usr/bin:/bin:${process.env.PATH ?? ''}` },
|
|
32
35
|
}).toString().trim();
|
|
33
36
|
ytdlpAvailable = true;
|
|
34
|
-
|
|
37
|
+
_ytLog.debug(`yt-dlp available: v${version}`);
|
|
35
38
|
}
|
|
36
39
|
catch {
|
|
37
|
-
|
|
40
|
+
_ytLog.debug('yt-dlp NOT available — falling back to HTTP extraction');
|
|
38
41
|
}
|
|
39
42
|
})();
|
|
40
43
|
// ---------------------------------------------------------------------------
|
|
@@ -26,24 +26,25 @@ import { normalizeActions } from '../../core/actions.js';
|
|
|
26
26
|
* Validate a URL from a request body. Sends a 400 response and returns false on failure.
|
|
27
27
|
* Returns true when the URL is valid and safe to use.
|
|
28
28
|
*/
|
|
29
|
-
function validateRequestUrl(url, res) {
|
|
29
|
+
function validateRequestUrl(url, req, res) {
|
|
30
|
+
const requestId = req.requestId;
|
|
30
31
|
if (!url || typeof url !== 'string') {
|
|
31
|
-
res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing or invalid "url" parameter' } });
|
|
32
|
+
res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing or invalid "url" parameter', hint: 'Provide a full URL including scheme, e.g. https://example.com', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId });
|
|
32
33
|
return false;
|
|
33
34
|
}
|
|
34
35
|
if (url.length > 2048) {
|
|
35
|
-
res.status(400).json({ success: false, error: { type: 'invalid_url', message: 'URL too long (max 2048 characters)' } });
|
|
36
|
+
res.status(400).json({ success: false, error: { type: 'invalid_url', message: 'URL too long (max 2048 characters)', hint: 'Shorten the URL or remove unnecessary query parameters', docs: 'https://webpeel.dev/docs/errors#invalid_url' }, requestId });
|
|
36
37
|
return false;
|
|
37
38
|
}
|
|
38
39
|
try {
|
|
39
40
|
const parsed = new URL(url);
|
|
40
41
|
if (!['http:', 'https:'].includes(parsed.protocol)) {
|
|
41
|
-
res.status(400).json({ success: false, error: { type: 'invalid_url', message: 'Only HTTP and HTTPS protocols are allowed' } });
|
|
42
|
+
res.status(400).json({ success: false, error: { type: 'invalid_url', message: 'Only HTTP and HTTPS protocols are allowed', hint: 'Ensure the URL starts with https:// or http://', docs: 'https://webpeel.dev/docs/errors#invalid_url' }, requestId });
|
|
42
43
|
return false;
|
|
43
44
|
}
|
|
44
45
|
}
|
|
45
46
|
catch {
|
|
46
|
-
res.status(400).json({ success: false, error: { type: 'invalid_url', message: 'Invalid URL format' } });
|
|
47
|
+
res.status(400).json({ success: false, error: { type: 'invalid_url', message: 'Invalid URL format', hint: 'Ensure the URL includes a scheme (https://) and a valid hostname', docs: 'https://webpeel.dev/docs/errors#invalid_url' }, requestId });
|
|
47
48
|
return false;
|
|
48
49
|
}
|
|
49
50
|
try {
|
|
@@ -51,7 +52,7 @@ function validateRequestUrl(url, res) {
|
|
|
51
52
|
}
|
|
52
53
|
catch (error) {
|
|
53
54
|
if (error instanceof SSRFError) {
|
|
54
|
-
res.status(400).json({ success: false, error: { type: 'ssrf_blocked', message: 'Cannot fetch localhost, private networks, or non-HTTP URLs' } });
|
|
55
|
+
res.status(400).json({ success: false, error: { type: 'ssrf_blocked', message: 'Cannot fetch localhost, private networks, or non-HTTP URLs', hint: 'Use a publicly reachable URL', docs: 'https://webpeel.dev/docs/errors#ssrf_blocked' }, requestId });
|
|
55
56
|
return false;
|
|
56
57
|
}
|
|
57
58
|
throw error;
|
|
@@ -93,11 +94,11 @@ async function handleFilmstrip(req, res, authStore) {
|
|
|
93
94
|
try {
|
|
94
95
|
const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
|
|
95
96
|
if (!ssUserId) {
|
|
96
|
-
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
|
|
97
|
+
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
|
|
97
98
|
return;
|
|
98
99
|
}
|
|
99
100
|
const { url, frames = 6, width, height, format = 'png', quality, waitFor, timeout, headers, cookies, stealth, } = req.body;
|
|
100
|
-
if (!validateRequestUrl(url, res))
|
|
101
|
+
if (!validateRequestUrl(url, req, res))
|
|
101
102
|
return;
|
|
102
103
|
if (frames !== undefined && (typeof frames !== 'number' || frames < 2 || frames > 12)) {
|
|
103
104
|
res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid frames: must be between 2 and 12' }, requestId: req.requestId });
|
|
@@ -190,11 +191,11 @@ async function handleAudit(req, res, authStore) {
|
|
|
190
191
|
try {
|
|
191
192
|
const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
|
|
192
193
|
if (!ssUserId) {
|
|
193
|
-
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
|
|
194
|
+
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
|
|
194
195
|
return;
|
|
195
196
|
}
|
|
196
197
|
const { url, width, height, format = 'jpeg', quality, selector, scrollThrough = false, waitFor, timeout } = req.body;
|
|
197
|
-
if (!validateRequestUrl(url, res))
|
|
198
|
+
if (!validateRequestUrl(url, req, res))
|
|
198
199
|
return;
|
|
199
200
|
if (format !== undefined && !['png', 'jpeg', 'jpg'].includes(format)) {
|
|
200
201
|
res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid format: must be "png", "jpeg", or "jpg"' }, requestId: req.requestId });
|
|
@@ -242,11 +243,11 @@ async function handleViewports(req, res, authStore) {
|
|
|
242
243
|
try {
|
|
243
244
|
const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
|
|
244
245
|
if (!ssUserId) {
|
|
245
|
-
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
|
|
246
|
+
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
|
|
246
247
|
return;
|
|
247
248
|
}
|
|
248
249
|
const { url, viewports, fullPage = false, format = 'jpeg', quality, scrollThrough = false, waitFor, timeout } = req.body;
|
|
249
|
-
if (!validateRequestUrl(url, res))
|
|
250
|
+
if (!validateRequestUrl(url, req, res))
|
|
250
251
|
return;
|
|
251
252
|
if (format !== undefined && !['png', 'jpeg', 'jpg'].includes(format)) {
|
|
252
253
|
res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid format: must be "png", "jpeg", or "jpg"' }, requestId: req.requestId });
|
|
@@ -304,11 +305,11 @@ async function handleDesignAuditHandler(req, res, authStore) {
|
|
|
304
305
|
try {
|
|
305
306
|
const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
|
|
306
307
|
if (!ssUserId) {
|
|
307
|
-
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
|
|
308
|
+
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
|
|
308
309
|
return;
|
|
309
310
|
}
|
|
310
311
|
const { url, rules, selector, width, height, waitFor, timeout } = req.body;
|
|
311
|
-
if (!validateRequestUrl(url, res))
|
|
312
|
+
if (!validateRequestUrl(url, req, res))
|
|
312
313
|
return;
|
|
313
314
|
if (rules !== undefined && typeof rules !== 'object') {
|
|
314
315
|
res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid "rules": must be an object' }, requestId: req.requestId });
|
|
@@ -347,11 +348,11 @@ async function handleDesignAnalysisHandler(req, res, authStore) {
|
|
|
347
348
|
try {
|
|
348
349
|
const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
|
|
349
350
|
if (!ssUserId) {
|
|
350
|
-
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
|
|
351
|
+
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
|
|
351
352
|
return;
|
|
352
353
|
}
|
|
353
354
|
const { url, selector, width, height, waitFor, timeout, stealth } = req.body;
|
|
354
|
-
if (!validateRequestUrl(url, res))
|
|
355
|
+
if (!validateRequestUrl(url, req, res))
|
|
355
356
|
return;
|
|
356
357
|
const startTime = Date.now();
|
|
357
358
|
const result = await takeDesignAnalysis(url, {
|
|
@@ -389,11 +390,11 @@ async function handleDesignMerged(req, res, authStore) {
|
|
|
389
390
|
try {
|
|
390
391
|
const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
|
|
391
392
|
if (!ssUserId) {
|
|
392
|
-
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
|
|
393
|
+
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
|
|
393
394
|
return;
|
|
394
395
|
}
|
|
395
396
|
const { url, rules, selector, width, height, waitFor, timeout, stealth } = req.body;
|
|
396
|
-
if (!validateRequestUrl(url, res))
|
|
397
|
+
if (!validateRequestUrl(url, req, res))
|
|
397
398
|
return;
|
|
398
399
|
if (rules !== undefined && typeof rules !== 'object') {
|
|
399
400
|
res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid "rules": must be an object' }, requestId: req.requestId });
|
|
@@ -444,16 +445,16 @@ async function handleDiff(req, res, authStore) {
|
|
|
444
445
|
try {
|
|
445
446
|
const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
|
|
446
447
|
if (!ssUserId) {
|
|
447
|
-
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
|
|
448
|
+
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
|
|
448
449
|
return;
|
|
449
450
|
}
|
|
450
451
|
// Accept both { url1, url2 } (legacy) and { url, compareUrl } (new mode param style)
|
|
451
452
|
const url1 = req.body.url1 ?? req.body.url;
|
|
452
453
|
const url2 = req.body.url2 ?? req.body.compareUrl;
|
|
453
454
|
const { width, height, fullPage = false, threshold, waitFor, timeout } = req.body;
|
|
454
|
-
if (!validateRequestUrl(url1, res))
|
|
455
|
+
if (!validateRequestUrl(url1, req, res))
|
|
455
456
|
return;
|
|
456
|
-
if (!validateRequestUrl(url2, res))
|
|
457
|
+
if (!validateRequestUrl(url2, req, res))
|
|
457
458
|
return;
|
|
458
459
|
if (url1 === url2) {
|
|
459
460
|
res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'url1 and url2 must be different URLs' }, requestId: req.requestId });
|
|
@@ -520,7 +521,7 @@ async function handleDesignCompare(req, res, authStore) {
|
|
|
520
521
|
try {
|
|
521
522
|
const userId = req.auth?.keyInfo?.accountId || req.user?.userId;
|
|
522
523
|
if (!userId) {
|
|
523
|
-
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
|
|
524
|
+
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
|
|
524
525
|
return;
|
|
525
526
|
}
|
|
526
527
|
// Accept body params (POST mode) or query params (GET /v1/design-compare)
|
|
@@ -532,13 +533,13 @@ async function handleDesignCompare(req, res, authStore) {
|
|
|
532
533
|
res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing required parameter "url"' }, requestId: req.requestId });
|
|
533
534
|
return;
|
|
534
535
|
}
|
|
535
|
-
if (!validateRequestUrl(url, res))
|
|
536
|
+
if (!validateRequestUrl(url, req, res))
|
|
536
537
|
return;
|
|
537
538
|
if (!ref || typeof ref !== 'string') {
|
|
538
539
|
res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing required parameter "compareUrl" (or "ref")' }, requestId: req.requestId });
|
|
539
540
|
return;
|
|
540
541
|
}
|
|
541
|
-
if (!validateRequestUrl(ref, res))
|
|
542
|
+
if (!validateRequestUrl(ref, req, res))
|
|
542
543
|
return;
|
|
543
544
|
if (url === ref) {
|
|
544
545
|
res.status(400).json({ success: false, error: { type: 'invalid_request', message: '"url" and "compareUrl" must be different URLs' }, requestId: req.requestId });
|
|
@@ -609,11 +610,11 @@ export function createScreenshotRouter(authStore) {
|
|
|
609
610
|
try {
|
|
610
611
|
const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
|
|
611
612
|
if (!ssUserId) {
|
|
612
|
-
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
|
|
613
|
+
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
|
|
613
614
|
return;
|
|
614
615
|
}
|
|
615
616
|
const { url, fullPage = false, width, height, format = 'png', quality, waitFor, timeout, actions, headers, cookies, stealth, scrollThrough = false, selector, } = req.body;
|
|
616
|
-
if (!validateRequestUrl(url, res))
|
|
617
|
+
if (!validateRequestUrl(url, req, res))
|
|
617
618
|
return;
|
|
618
619
|
if (format !== undefined && !['png', 'jpeg', 'jpg'].includes(format)) {
|
|
619
620
|
res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid format: must be "png", "jpeg", or "jpg"' }, requestId: req.requestId });
|
|
@@ -749,11 +750,11 @@ export function createScreenshotRouter(authStore) {
|
|
|
749
750
|
try {
|
|
750
751
|
const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
|
|
751
752
|
if (!ssUserId) {
|
|
752
|
-
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
|
|
753
|
+
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
|
|
753
754
|
return;
|
|
754
755
|
}
|
|
755
756
|
const { url, rules, selector } = req.body;
|
|
756
|
-
if (!validateRequestUrl(url, res))
|
|
757
|
+
if (!validateRequestUrl(url, req, res))
|
|
757
758
|
return;
|
|
758
759
|
const startTime = Date.now();
|
|
759
760
|
const [viewportsResult, auditResult] = await Promise.all([
|
|
@@ -801,7 +802,7 @@ export function createScreenshotRouter(authStore) {
|
|
|
801
802
|
router.get('/v1/design-compare', async (req, res) => {
|
|
802
803
|
const userId = req.auth?.keyInfo?.accountId || req.user?.userId;
|
|
803
804
|
if (!userId) {
|
|
804
|
-
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
|
|
805
|
+
res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
|
|
805
806
|
return;
|
|
806
807
|
}
|
|
807
808
|
// Validate ref query param (url is validated inside handleDesignCompare)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "webpeel",
|
|
3
|
-
"version": "0.20.
|
|
3
|
+
"version": "0.20.21",
|
|
4
4
|
"description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
|
|
5
5
|
"author": "Jake Liu",
|
|
6
6
|
"license": "AGPL-3.0-only",
|