webpeel 0.20.19 → 0.20.20

package/dist/core/youtube.js

@@ -15,11 +15,14 @@ import { join } from 'node:path';
 import { fetchTranscript as ytpFetchTranscript } from 'youtube-transcript-plus';
 import { simpleFetch } from './fetcher.js';
 import { getBrowser, getRandomUserAgent, applyStealthScripts } from './browser-pool.js';
+import { createLogger } from './logger.js';
 // ---------------------------------------------------------------------------
 // yt-dlp startup diagnostics
 // ---------------------------------------------------------------------------
+const _ytLog = createLogger('youtube');
 // Check yt-dlp availability on startup.
 // Skipped in test environments (VITEST) to avoid interfering with mocked paths.
+// Uses logger.debug (→ stderr) so it never pollutes stdout JSON output when piped.
 let ytdlpAvailable = false;
 (async () => {
     if (process.env.VITEST)
@@ -31,10 +34,10 @@ let ytdlpAvailable = false;
             env: { ...process.env, PATH: `/usr/local/bin:/usr/bin:/bin:${process.env.PATH ?? ''}` },
         }).toString().trim();
         ytdlpAvailable = true;
-        console.log(`[webpeel] [youtube] yt-dlp available: v${version}`);
+        _ytLog.debug(`yt-dlp available: v${version}`);
     }
     catch {
-        console.log('[webpeel] [youtube] yt-dlp NOT available — falling back to HTTP extraction');
+        _ytLog.debug('yt-dlp NOT available — falling back to HTTP extraction');
     }
 })();
 // ---------------------------------------------------------------------------

package/dist/server/routes/screenshot.js

@@ -26,24 +26,25 @@ import { normalizeActions } from '../../core/actions.js';
  * Validate a URL from a request body. Sends a 400 response and returns false on failure.
  * Returns true when the URL is valid and safe to use.
  */
-function validateRequestUrl(url, res) {
+function validateRequestUrl(url, req, res) {
+    const requestId = req.requestId;
     if (!url || typeof url !== 'string') {
-        res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing or invalid "url" parameter' } });
+        res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing or invalid "url" parameter', hint: 'Provide a full URL including scheme, e.g. https://example.com', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId });
         return false;
     }
     if (url.length > 2048) {
-        res.status(400).json({ success: false, error: { type: 'invalid_url', message: 'URL too long (max 2048 characters)' } });
+        res.status(400).json({ success: false, error: { type: 'invalid_url', message: 'URL too long (max 2048 characters)', hint: 'Shorten the URL or remove unnecessary query parameters', docs: 'https://webpeel.dev/docs/errors#invalid_url' }, requestId });
         return false;
     }
     try {
         const parsed = new URL(url);
         if (!['http:', 'https:'].includes(parsed.protocol)) {
-            res.status(400).json({ success: false, error: { type: 'invalid_url', message: 'Only HTTP and HTTPS protocols are allowed' } });
+            res.status(400).json({ success: false, error: { type: 'invalid_url', message: 'Only HTTP and HTTPS protocols are allowed', hint: 'Ensure the URL starts with https:// or http://', docs: 'https://webpeel.dev/docs/errors#invalid_url' }, requestId });
             return false;
         }
     }
     catch {
-        res.status(400).json({ success: false, error: { type: 'invalid_url', message: 'Invalid URL format' } });
+        res.status(400).json({ success: false, error: { type: 'invalid_url', message: 'Invalid URL format', hint: 'Ensure the URL includes a scheme (https://) and a valid hostname', docs: 'https://webpeel.dev/docs/errors#invalid_url' }, requestId });
         return false;
     }
     try {
@@ -51,7 +52,7 @@ function validateRequestUrl(url, res) {
     }
     catch (error) {
         if (error instanceof SSRFError) {
-            res.status(400).json({ success: false, error: { type: 'ssrf_blocked', message: 'Cannot fetch localhost, private networks, or non-HTTP URLs' } });
+            res.status(400).json({ success: false, error: { type: 'ssrf_blocked', message: 'Cannot fetch localhost, private networks, or non-HTTP URLs', hint: 'Use a publicly reachable URL', docs: 'https://webpeel.dev/docs/errors#ssrf_blocked' }, requestId });
             return false;
         }
         throw error;
@@ -93,11 +94,11 @@ async function handleFilmstrip(req, res, authStore) {
     try {
         const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
         if (!ssUserId) {
-            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
+            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
             return;
         }
         const { url, frames = 6, width, height, format = 'png', quality, waitFor, timeout, headers, cookies, stealth, } = req.body;
-        if (!validateRequestUrl(url, res))
+        if (!validateRequestUrl(url, req, res))
             return;
         if (frames !== undefined && (typeof frames !== 'number' || frames < 2 || frames > 12)) {
             res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid frames: must be between 2 and 12' }, requestId: req.requestId });
@@ -190,11 +191,11 @@ async function handleAudit(req, res, authStore) {
     try {
         const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
         if (!ssUserId) {
-            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
+            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
             return;
         }
         const { url, width, height, format = 'jpeg', quality, selector, scrollThrough = false, waitFor, timeout } = req.body;
-        if (!validateRequestUrl(url, res))
+        if (!validateRequestUrl(url, req, res))
             return;
         if (format !== undefined && !['png', 'jpeg', 'jpg'].includes(format)) {
             res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid format: must be "png", "jpeg", or "jpg"' }, requestId: req.requestId });
@@ -242,11 +243,11 @@ async function handleViewports(req, res, authStore) {
     try {
         const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
         if (!ssUserId) {
-            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
+            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
             return;
         }
         const { url, viewports, fullPage = false, format = 'jpeg', quality, scrollThrough = false, waitFor, timeout } = req.body;
-        if (!validateRequestUrl(url, res))
+        if (!validateRequestUrl(url, req, res))
             return;
         if (format !== undefined && !['png', 'jpeg', 'jpg'].includes(format)) {
             res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid format: must be "png", "jpeg", or "jpg"' }, requestId: req.requestId });
@@ -304,11 +305,11 @@ async function handleDesignAuditHandler(req, res, authStore) {
     try {
         const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
         if (!ssUserId) {
-            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
+            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
             return;
         }
         const { url, rules, selector, width, height, waitFor, timeout } = req.body;
-        if (!validateRequestUrl(url, res))
+        if (!validateRequestUrl(url, req, res))
             return;
         if (rules !== undefined && typeof rules !== 'object') {
             res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid "rules": must be an object' }, requestId: req.requestId });
@@ -347,11 +348,11 @@ async function handleDesignAnalysisHandler(req, res, authStore) {
     try {
         const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
         if (!ssUserId) {
-            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
+            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
             return;
         }
         const { url, selector, width, height, waitFor, timeout, stealth } = req.body;
-        if (!validateRequestUrl(url, res))
+        if (!validateRequestUrl(url, req, res))
             return;
         const startTime = Date.now();
         const result = await takeDesignAnalysis(url, {
@@ -389,11 +390,11 @@ async function handleDesignMerged(req, res, authStore) {
     try {
         const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
         if (!ssUserId) {
-            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
+            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
             return;
         }
         const { url, rules, selector, width, height, waitFor, timeout, stealth } = req.body;
-        if (!validateRequestUrl(url, res))
+        if (!validateRequestUrl(url, req, res))
             return;
         if (rules !== undefined && typeof rules !== 'object') {
             res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid "rules": must be an object' }, requestId: req.requestId });
@@ -444,16 +445,16 @@ async function handleDiff(req, res, authStore) {
     try {
         const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
         if (!ssUserId) {
-            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
+            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
             return;
         }
         // Accept both { url1, url2 } (legacy) and { url, compareUrl } (new mode param style)
         const url1 = req.body.url1 ?? req.body.url;
         const url2 = req.body.url2 ?? req.body.compareUrl;
         const { width, height, fullPage = false, threshold, waitFor, timeout } = req.body;
-        if (!validateRequestUrl(url1, res))
+        if (!validateRequestUrl(url1, req, res))
             return;
-        if (!validateRequestUrl(url2, res))
+        if (!validateRequestUrl(url2, req, res))
             return;
         if (url1 === url2) {
             res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'url1 and url2 must be different URLs' }, requestId: req.requestId });
@@ -520,7 +521,7 @@ async function handleDesignCompare(req, res, authStore) {
     try {
         const userId = req.auth?.keyInfo?.accountId || req.user?.userId;
         if (!userId) {
-            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
+            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
             return;
         }
         // Accept body params (POST mode) or query params (GET /v1/design-compare)
@@ -532,13 +533,13 @@ async function handleDesignCompare(req, res, authStore) {
             res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing required parameter "url"' }, requestId: req.requestId });
             return;
         }
-        if (!validateRequestUrl(url, res))
+        if (!validateRequestUrl(url, req, res))
             return;
         if (!ref || typeof ref !== 'string') {
             res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing required parameter "compareUrl" (or "ref")' }, requestId: req.requestId });
             return;
         }
-        if (!validateRequestUrl(ref, res))
+        if (!validateRequestUrl(ref, req, res))
             return;
         if (url === ref) {
             res.status(400).json({ success: false, error: { type: 'invalid_request', message: '"url" and "compareUrl" must be different URLs' }, requestId: req.requestId });
@@ -609,11 +610,11 @@ export function createScreenshotRouter(authStore) {
         try {
             const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
             if (!ssUserId) {
-                res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
+                res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
                 return;
             }
             const { url, fullPage = false, width, height, format = 'png', quality, waitFor, timeout, actions, headers, cookies, stealth, scrollThrough = false, selector, } = req.body;
-            if (!validateRequestUrl(url, res))
+            if (!validateRequestUrl(url, req, res))
                 return;
             if (format !== undefined && !['png', 'jpeg', 'jpg'].includes(format)) {
                 res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid format: must be "png", "jpeg", or "jpg"' }, requestId: req.requestId });
@@ -749,11 +750,11 @@ export function createScreenshotRouter(authStore) {
         try {
             const ssUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
             if (!ssUserId) {
-                res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
+                res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
                 return;
             }
             const { url, rules, selector } = req.body;
-            if (!validateRequestUrl(url, res))
+            if (!validateRequestUrl(url, req, res))
                 return;
             const startTime = Date.now();
             const [viewportsResult, auditResult] = await Promise.all([
@@ -801,7 +802,7 @@ export function createScreenshotRouter(authStore) {
     router.get('/v1/design-compare', async (req, res) => {
         const userId = req.auth?.keyInfo?.accountId || req.user?.userId;
         if (!userId) {
-            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' } });
+            res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'API key required.', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
             return;
         }
         // Validate ref query param (url is validated inside handleDesignCompare)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.20.19",
+  "version": "0.20.20",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",