webpeel 0.20.9 → 0.20.11

package/dist/core/youtube.js

@@ -224,8 +224,15 @@ export async function getYouTubeTranscript(url, options = {}) {
     const preferredLang = options.language ?? 'en';
     const videoUrl = `https://www.youtube.com/watch?v=${videoId}`;
     // --- Path 1: Try simpleFetch (fast, no browser overhead) ---
+    // YouTube serves consent/challenge pages to server IPs without cookies.
+    // Setting SOCS consent cookie bypasses this — same approach as youtube-transcript npm.
+    const ytUserAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
+    const ytHeaders = {
+        'Cookie': 'SOCS=CAISNQgDEitib3FfaWRlbnRpdHlmcm9udGVuZHVpc2VydmVyXzIwMjQwNTE1LjA3X3AxGgJlbiADGgYIgLv3tQY; CONSENT=PENDING+987',
+        'Accept-Language': 'en-US,en;q=0.9',
+    };
     try {
-        const fetchResult = await simpleFetch(videoUrl, undefined, 15000);
+        const fetchResult = await simpleFetch(videoUrl, ytUserAgent, 15000, ytHeaders);
         const html = fetchResult.html;
         if (!html.includes('ytInitialPlayerResponse') && !html.includes('ytInitialData')) {
             throw new Error('YouTube served non-video page (likely challenge/consent)');
@@ -245,11 +252,12 @@ export async function getYouTubeTranscript(url, options = {}) {
             throw new Error('No captions available');
         const availableLanguages = captionTracks.map(t => t.languageCode);
         const selectedTrack = selectBestTrack(captionTracks, preferredLang);
-        const captionXml = await fetchCaptionXml(selectedTrack.baseUrl);
+        // Pass same cookies + user-agent to caption fetch — URL is session-locked
+        const captionXml = await fetchCaptionXml(selectedTrack.baseUrl, ytUserAgent, ytHeaders);
         const segments = parseCaptionXml(captionXml);
         if (segments.length === 0) {
             // Caption URL returned empty content (common when ip=0.0.0.0 in signature)
-            // Fall through to browser intercept path
+            // Fall through to yt-dlp / browser intercept path
             throw new Error('Caption XML returned empty — session-locked URL');
         }
         const fullText = segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim();
@@ -671,10 +679,10 @@ function selectBestTrack(tracks, preferredLang) {
 }
 /**
  * Fetch the caption XML from YouTube's timedtext API.
- * Used only in the simpleFetch path (no bot detection).
+ * Must use same cookies/UA as the page fetch — URLs are session-locked.
  */
-async function fetchCaptionXml(baseUrl) {
-    const result = await simpleFetch(baseUrl, undefined, 15000);
+async function fetchCaptionXml(baseUrl, userAgent, headers) {
+    const result = await simpleFetch(baseUrl, userAgent, 10000, headers);
     return result.html;
 }
 /**

package/dist/server/app.js

@@ -94,6 +94,7 @@ export function createApp(config = {}) {
     app.use((req, res, next) => {
         const path = req.path;
         let timeoutMs = 30000; // 30s default
+        const urlParam = req.query?.url || '';
         if (path.includes('/crawl') || path.includes('/map'))
             timeoutMs = 300000; // 5min for crawls
         else if (path.includes('/batch'))
@@ -102,6 +103,8 @@ export function createApp(config = {}) {
             timeoutMs = 60000; // 1min for screenshots
         else if (req.query?.render === 'true')
             timeoutMs = 60000; // 1min for rendered fetches
+        else if (urlParam.includes('youtube.com') || urlParam.includes('youtu.be'))
+            timeoutMs = 45000; // 45s for YouTube (transcript extraction)
         req.setTimeout(timeoutMs);
         res.setTimeout(timeoutMs, () => {
             if (!res.headersSent) {

package/dist/server/routes/fetch.js

@@ -516,6 +516,8 @@ export function createFetchRouter(authStore) {
             // SECURITY: Sanitize error messages to prevent information disclosure
             if (err.code) {
                 // WebPeelError from core library - safe to expose with helpful context
+                if (res.headersSent)
+                    return; // Timeout middleware already responded
                 const safeMessage = err.message.replace(/[<>"']/g, ''); // Remove HTML chars
                 const statusCode = err.code === 'TIMEOUT' ? 504
                     : err.code === 'BLOCKED' ? 403
@@ -540,6 +542,8 @@ export function createFetchRouter(authStore) {
             else {
                 // Unexpected error - generic message only
                 console.error('Fetch error:', err); // Log full error server-side
+                if (res.headersSent)
+                    return; // Timeout middleware already responded
                 res.status(500).json({
                     success: false,
                     error: {
@@ -1000,6 +1004,8 @@ export function createFetchRouter(authStore) {
         catch (error) {
             const err = error;
             console.error('POST fetch/scrape error:', err);
+            if (res.headersSent)
+                return; // Timeout middleware already responded
             if (err.code) {
                 const safeMessage = err.message.replace(/[<>"']/g, '');
                 const statusCode = err.code === 'TIMEOUT' ? 504

package/dist/server/routes/health.js

@@ -4,13 +4,31 @@
  * so it's never blocked by rate limiting (Render hits it every ~30s).
  */
 import { Router } from 'express';
+import { readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
 const startTime = Date.now();
+// Read version once at startup
+let version = 'unknown';
+try {
+    const pkgPath = join(dirname(fileURLToPath(import.meta.url)), '..', '..', '..', 'package.json');
+    version = JSON.parse(readFileSync(pkgPath, 'utf-8')).version;
+}
+catch {
+    // Fallback for bundled/Docker environments
+    try {
+        const altPath = join(process.cwd(), 'package.json');
+        version = JSON.parse(readFileSync(altPath, 'utf-8')).version;
+    }
+    catch { /* keep 'unknown' */ }
+}
 export function createHealthRouter() {
     const router = Router();
     router.get('/health', (_req, res) => {
         const uptime = Math.floor((Date.now() - startTime) / 1000);
         res.json({
             status: 'healthy',
+            version,
             uptime,
             timestamp: new Date().toISOString(),
         });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.20.9",
+  "version": "0.20.11",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",