webpeel 0.20.13 → 0.20.17

package/dist/core/domain-extractors.js

@@ -1165,8 +1165,8 @@ async function youtubeExtractor(_html, url) {
         ]);
     }
     // Run transcript fetch and oEmbed fetch in parallel
-    // Browser-rendered fetch takes ~10s — use 15s timeout so browser has time to render
-    const transcriptPromise = withTimeout(getYouTubeTranscript(url), 15000);
+    // Proxy-based extraction takes 2-5s, but retry logic may need more time
+    const transcriptPromise = withTimeout(getYouTubeTranscript(url), 30000);
     const oembedPromise = fetchJson(`https://www.youtube.com/oembed?url=${encodeURIComponent(url)}&format=json`);
     const noembedPromise = fetchJson(`https://noembed.com/embed?url=${encodeURIComponent(url)}`).catch(() => null);
     const [transcriptResult, oembedResult, noembedResult] = await Promise.allSettled([
@@ -1233,7 +1233,9 @@ async function youtubeExtractor(_html, url) {
         parts.push(headerLine);
         // Summary section
         if (transcript.summary && hasTranscript) {
-            parts.push(`## Summary\n\n${transcript.summary}`);
+            let summaryText = transcript.summary;
+            summaryText = summaryText.replace(/([.!?])\s+(?=[A-Z])/g, '$1\n\n');
+            parts.push(`## Summary\n\n${summaryText}`);
         }
         else if (!hasTranscript && transcript.fullText) {
             parts.push(`## Description\n\n${transcript.fullText}`);
@@ -1249,8 +1251,14 @@ async function youtubeExtractor(_html, url) {
             parts.push(`## Chapters\n\n${chLines}`);
         }
         // Full Transcript section (only if we have real transcript segments)
+        // Add intelligent paragraph breaks for readability
         if (hasTranscript) {
-            parts.push(`## Full Transcript\n\n${transcript.fullText}`);
+            let readableText = transcript.fullText;
+            // Break into paragraphs: after sentence-ending punctuation followed by a capital letter
+            readableText = readableText.replace(/([.!?])\s+(?=[A-Z])/g, '$1\n\n');
+            // Collapse any triple+ newlines
+            readableText = readableText.replace(/\n{3,}/g, '\n\n');
+            parts.push(`## Full Transcript\n\n${readableText}`);
         }
         const cleanContent = parts.join('\n\n');
         return { domain: 'youtube.com', type: 'video', structured, cleanContent };

package/dist/core/youtube.js

@@ -6,12 +6,38 @@
  * track URLs, fetch the timedtext XML, and return structured transcript data.
  */
 import { execFile } from 'node:child_process';
+import * as http from 'node:http';
+import * as https from 'node:https';
+import * as tls from 'node:tls';
 import { readFile, unlink } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
+import { fetchTranscript as ytpFetchTranscript } from 'youtube-transcript-plus';
 import { simpleFetch } from './fetcher.js';
 import { getBrowser, getRandomUserAgent, applyStealthScripts } from './browser-pool.js';
 // ---------------------------------------------------------------------------
+// yt-dlp startup diagnostics
+// ---------------------------------------------------------------------------
+// Check yt-dlp availability on startup.
+// Skipped in test environments (VITEST) to avoid interfering with mocked paths.
+let ytdlpAvailable = false;
+(async () => {
+    if (process.env.VITEST)
+        return;
+    try {
+        const { execFileSync } = await import('node:child_process');
+        const version = execFileSync('yt-dlp', ['--version'], {
+            timeout: 5000,
+            env: { ...process.env, PATH: `/usr/local/bin:/usr/bin:/bin:${process.env.PATH ?? ''}` },
+        }).toString().trim();
+        ytdlpAvailable = true;
+        console.log(`[webpeel] [youtube] yt-dlp available: v${version}`);
+    }
+    catch {
+        console.log('[webpeel] [youtube] yt-dlp NOT available — falling back to HTTP extraction');
+    }
+})();
+// ---------------------------------------------------------------------------
 // URL parsing
 // ---------------------------------------------------------------------------
 /**
@@ -208,6 +234,217 @@ export function extractSummary(fullText) {
     return words.slice(0, 200).join(' ') + '...';
 }
 // ---------------------------------------------------------------------------
+// Proxy-based InnerTube transcript extraction
+// ---------------------------------------------------------------------------
+// Webshare residential proxy config — reads from env vars on Render.
+// Locally, falls back to direct fetch (residential IP already works).
+const PROXY_HOST = process.env.WEBSHARE_PROXY_HOST || 'p.webshare.io';
+const PROXY_BASE_PORT = parseInt(process.env.WEBSHARE_PROXY_PORT || '10000', 10);
+const PROXY_USER = process.env.WEBSHARE_PROXY_USER || '';
+const PROXY_PASS = process.env.WEBSHARE_PROXY_PASS || '';
+// With paid Webshare backbone plan, each US slot has its own port:
+// slot N → port (PROXY_BASE_PORT + N - 1), username: USER-US-N
+const PROXY_MAX_US_SLOTS = parseInt(process.env.WEBSHARE_PROXY_SLOTS || '44744', 10);
+function isProxyConfigured() {
+    return !!(PROXY_USER && PROXY_PASS);
+}
+/**
+ * Make an HTTP(S) request through the Webshare CONNECT proxy with a specific
+ * slotted username (e.g. "argtnlhz-5"). This ensures both the /player call
+ * and the caption XML fetch go through the same residential IP.
+ */
+function proxyRequestSlotted(slottedUser, proxyPort, targetUrl, opts = {}) {
+    const url = new URL(targetUrl);
+    const timeout = opts.timeoutMs ?? 20000;
+    return new Promise((resolve, reject) => {
+        const proxyAuth = Buffer.from(`${slottedUser}:${PROXY_PASS}`).toString('base64');
+        const proxyReq = http.request({
+            host: PROXY_HOST,
+            port: proxyPort,
+            method: 'CONNECT',
+            path: `${url.hostname}:443`,
+            headers: { 'Proxy-Authorization': `Basic ${proxyAuth}` },
+        });
+        const timer = setTimeout(() => {
+            proxyReq.destroy();
+            reject(new Error('Proxy request timed out'));
+        }, timeout);
+        proxyReq.on('connect', (res, socket) => {
+            if (res.statusCode !== 200) {
+                clearTimeout(timer);
+                socket.destroy();
+                reject(new Error(`Proxy CONNECT failed: ${res.statusCode}`));
+                return;
+            }
+            const tlsSocket = tls.connect({ host: url.hostname, socket, servername: url.hostname }, () => {
+                const reqHeaders = {
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                    'Accept-Language': 'en-US,en;q=0.9',
+                    'Cookie': 'CONSENT=YES+; SOCS=CAI',
+                    ...(opts.headers ?? {}),
+                };
+                const req = https.request({
+                    hostname: url.hostname,
+                    path: url.pathname + url.search,
+                    method: opts.method ?? 'GET',
+                    createConnection: () => tlsSocket,
+                    headers: reqHeaders,
+                }, (response) => {
+                    let data = '';
+                    response.on('data', (chunk) => {
+                        data += chunk;
+                    });
+                    response.on('end', () => {
+                        clearTimeout(timer);
+                        resolve({ status: response.statusCode ?? 0, body: data });
+                    });
+                });
+                req.on('error', (e) => {
+                    clearTimeout(timer);
+                    reject(e);
+                });
+                if (opts.body)
+                    req.write(opts.body);
+                req.end();
+            });
+            tlsSocket.on('error', (e) => {
+                clearTimeout(timer);
+                reject(e);
+            });
+        });
+        proxyReq.on('error', (e) => {
+            clearTimeout(timer);
+            reject(e);
+        });
+        proxyReq.end();
+    });
+}
+/**
+ * Fetch YouTube transcript via InnerTube /player API through Webshare proxy.
+ *
+ * This replicates the approach used by the Python `youtube-transcript-api` library:
+ * 1. POST to /youtubei/v1/player with ANDROID client context
+ * 2. Get caption track URLs WITHOUT the `exp=xpe` parameter
+ * 3. Fetch caption XML from those clean URLs (returns actual data, not 0 bytes)
+ *
+ * All requests go through the residential proxy to bypass YouTube's cloud IP blocking.
+ */
+async function getTranscriptViaProxy(videoId, preferredLang) {
+    // Try multiple proxy slots from the 44K+ US residential pool.
+    // Pick random slots across the pool for even distribution and to avoid
+    // rate-limited IPs. Try up to MAX_RETRIES different slots.
+    const MAX_RETRIES = 5;
+    const usedSlots = new Set();
+    const INNERTUBE_API_KEY = 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8';
+    for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+        // Pick a random US slot we haven't tried yet
+        let slot;
+        do {
+            slot = Math.floor(Math.random() * PROXY_MAX_US_SLOTS) + 1;
+        } while (usedSlots.has(slot) && usedSlots.size < PROXY_MAX_US_SLOTS);
+        usedSlots.add(slot);
+        const proxyUser = `${PROXY_USER}-US-${slot}`;
+        const proxyPort = PROXY_BASE_PORT + slot - 1;
+        const doProxyRequest = (url, opts = {}) => proxyRequestSlotted(proxyUser, proxyPort, url, opts);
+        try {
+            // Step 1: Call InnerTube /player with ANDROID client
+            // ANDROID client returns caption URLs WITHOUT exp=xpe (avoids 0-byte responses).
+            const playerResp = await doProxyRequest(`https://www.youtube.com/youtubei/v1/player?key=${INNERTUBE_API_KEY}`, {
+                method: 'POST',
+                body: JSON.stringify({
+                    context: { client: { clientName: 'ANDROID', clientVersion: '20.10.38' } },
+                    videoId,
+                }),
+                headers: { 'Content-Type': 'application/json' },
+            });
+            if (playerResp.status !== 200) {
+                console.log(`[webpeel] [youtube] Proxy US-${slot} (port ${proxyPort}): /player returned ${playerResp.status}`);
+                continue;
+            }
+            const playerData = JSON.parse(playerResp.body);
+            const captionTracks = playerData?.captions?.playerCaptionsTracklistRenderer?.captionTracks;
+            if (!captionTracks || captionTracks.length === 0) {
+                console.log(`[webpeel] [youtube] Proxy US-${slot} (port ${proxyPort}): no caption tracks`);
+                continue;
+            }
+            // Pick best matching language track
+            let track = captionTracks.find((t) => t.languageCode === preferredLang);
+            if (!track) {
+                track = captionTracks.find((t) => t.languageCode === 'en') ?? captionTracks[0];
+            }
+            const captionUrl = track.baseUrl;
+            if (captionUrl.includes('exp=xpe')) {
+                console.log(`[webpeel] [youtube] Proxy US-${slot} (port ${proxyPort}): caption URL has exp=xpe, skipping`);
+                continue;
+            }
+            // Step 2: Fetch caption XML through the SAME proxy slot (same residential IP)
+            const capResp = await doProxyRequest(captionUrl);
+            if (!capResp.body ||
+                capResp.body.length === 0 ||
+                capResp.status === 429 ||
+                capResp.body.includes('<title>Sorry...</title>')) {
+                console.log(`[webpeel] [youtube] Proxy US-${slot} (port ${proxyPort}): caption XML failed (status=${capResp.status}, bytes=${capResp.body?.length ?? 0})`);
+                continue; // Try next slot
+            }
+            // Parse XML segments — handles both <text start="" dur=""> and <p t="" d=""> formats
+            const xmlSegments = [
+                ...capResp.body.matchAll(/<(?:text|p)\s[^>]*?(?:start|t)="([^"]*)"[^>]*?(?:dur|d)="([^"]*)"[^>]*>([\s\S]*?)<\/(?:text|p)>/g),
+            ];
+            if (xmlSegments.length === 0) {
+                console.log(`[webpeel] [youtube] Proxy US-${slot} (port ${proxyPort}): no segments parsed from XML`);
+                continue;
+            }
+            const segments = xmlSegments
+                .map((m) => ({
+                text: decodeHtmlEntities(m[3].replace(/<[^>]+>/g, '').replace(/\n/g, ' ').trim()),
+                start: parseFloat(m[1]) / (m[1].includes('.') ? 1 : 1000),
+                duration: parseFloat(m[2]) / (m[2].includes('.') ? 1 : 1000),
+            }))
+                .filter((s) => s.text.length > 0);
+            if (segments.length === 0)
+                continue;
+            // Extract metadata from player response
+            const vd = playerData.videoDetails ?? {};
+            const mf = playerData.microformat?.playerMicroformatRenderer ?? {};
+            const title = vd.title ?? '';
+            const channel = vd.author ?? '';
+            const lengthSeconds = parseInt(vd.lengthSeconds ?? mf.lengthSeconds ?? '0', 10);
+            const description = (vd.shortDescription ?? mf.description?.simpleText ?? '').trim();
+            const publishDate = mf.publishDate ?? mf.uploadDate ?? '';
+            const availableLanguages = captionTracks.map((t) => t.languageCode);
+            const fullText = segments.map((s) => s.text).join(' ').replace(/\s+/g, ' ').trim();
+            const wordCount = fullText.split(/\s+/).filter(Boolean).length;
+            const chapters = parseChaptersFromDescription(description);
+            const keyPoints = extractKeyPoints(segments, chapters, lengthSeconds);
+            const summary = extractSummary(fullText);
+            console.log(`[webpeel] [youtube] Proxy slot ${slot} success: ${segments.length} segments, ${wordCount} words`);
+            return {
+                videoId,
+                title,
+                channel,
+                duration: formatDuration(lengthSeconds),
+                language: track.languageCode ?? preferredLang,
+                segments,
+                fullText,
+                availableLanguages,
+                description,
+                publishDate,
+                chapters: chapters.length > 0 ? chapters : undefined,
+                keyPoints: keyPoints.length > 0 ? keyPoints : undefined,
+                summary,
+                wordCount,
+            };
+        }
+        catch (err) {
+            console.log(`[webpeel] [youtube] Proxy slot ${slot} error:`, err?.message);
+            continue;
+        }
+    }
+    // All slots exhausted
+    console.log('[webpeel] [youtube] All proxy slots exhausted');
+    return null;
+}
+// ---------------------------------------------------------------------------
 // Transcript extraction
 // ---------------------------------------------------------------------------
 /**
@@ -223,17 +460,149 @@ export async function getYouTubeTranscript(url, options = {}) {
     }
     const preferredLang = options.language ?? 'en';
     const videoUrl = `https://www.youtube.com/watch?v=${videoId}`;
-    // --- Path 1: Try simpleFetch (fast, no browser overhead) ---
-    // YouTube serves consent/challenge pages to server IPs without cookies.
-    // Setting SOCS consent cookie bypasses this — same approach as youtube-transcript npm.
+    // --- Path P: Proxy-based InnerTube (primary for cloud servers) ---
+    // Uses Webshare residential proxy + ANDROID InnerTube /player API.
+    // This is the approach used by every major YouTube transcript service
+    // (youtubetotranscript.com, youtube-transcript.io, etc.)
+    if (!process.env.VITEST && isProxyConfigured()) {
+        console.log('[webpeel] [youtube] Trying path P: proxy-based InnerTube (residential proxy)');
+        try {
+            const proxyResult = await getTranscriptViaProxy(videoId, preferredLang);
+            if (proxyResult && proxyResult.segments.length > 0) {
+                console.log(`[webpeel] [youtube] Path P success: ${proxyResult.segments.length} segments, ${proxyResult.wordCount} words`);
+                return proxyResult;
+            }
+            console.log('[webpeel] [youtube] Path P returned empty/null, falling through');
+        }
+        catch (err) {
+            console.log('[webpeel] [youtube] Path P failed:', err?.message);
+        }
+    }
+    // --- Path 0: youtube-transcript-plus (fastest — uses InnerTube API, ~1s) ---
+    // This library calls YouTube's internal InnerTube API directly via POST request,
+    // bypassing the IP-locked timedtext XML URLs. Works reliably from cloud servers.
+    // Skip in test mode — tests use mocked HTTP, but this path makes real InnerTube calls.
+    if (!process.env.VITEST) {
+        console.log('[webpeel] [youtube] Trying path 0: youtube-transcript-plus (InnerTube API)');
+        try {
+            const ytpSegments = await ytpFetchTranscript(videoId, { lang: preferredLang });
+            if (ytpSegments && ytpSegments.length > 0) {
+                // We have transcript segments — now fetch page metadata (title, channel, etc.)
+                let title = '', channel = '', lengthSeconds = 0, description = '', publishDate = '';
+                let availableLanguages = [preferredLang];
+                try {
+                    const metaResp = await fetch(videoUrl, {
+                        headers: {
+                            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+                            'Cookie': 'SOCS=CAISNQgDEitib3FfaWRlbnRpdHlmcm9udGVuZHVpc2VydmVyXzIwMjQwNTE1LjA3X3AxGgJlbiADGgYIgLv3tQY; CONSENT=PENDING+987',
+                        },
+                        signal: AbortSignal.timeout(8000),
+                    });
+                    const html = await metaResp.text();
+                    const pr = extractPlayerResponse(html);
+                    if (pr) {
+                        const vd = pr.videoDetails ?? {};
+                        const mf = pr.microformat?.playerMicroformatRenderer ?? {};
+                        title = vd.title ?? '';
+                        channel = vd.author ?? '';
+                        lengthSeconds = parseInt(vd.lengthSeconds ?? mf.lengthSeconds ?? '0', 10);
+                        description = (vd.shortDescription ?? mf.description?.simpleText ?? '').trim();
+                        publishDate = mf.publishDate ?? mf.uploadDate ?? '';
+                        const tracks = extractCaptionTracks(pr);
+                        if (tracks.length > 0)
+                            availableLanguages = tracks.map(t => t.languageCode);
+                    }
+                }
+                catch { /* metadata fetch failed — segments are enough */ }
+                // Convert youtube-transcript-plus format to our format
+                const segments = ytpSegments.map(s => ({
+                    text: decodeHtmlEntities((s.text ?? '').replace(/\n/g, ' ').trim()),
+                    start: (s.offset ?? 0) / 1000, // offset is in ms
+                    duration: (s.duration ?? 0) / 1000,
+                })).filter(s => s.text.length > 0);
+                const fullText = segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim();
+                const wordCount = fullText.split(/\s+/).filter(Boolean).length;
+                const chapters = parseChaptersFromDescription(description);
+                const keyPoints = extractKeyPoints(segments, chapters, lengthSeconds);
+                const summary = extractSummary(fullText);
+                console.log(`[webpeel] [youtube] Path 0 success: ${segments.length} segments, ${wordCount} words`);
+                return {
+                    videoId,
+                    title,
+                    channel,
+                    duration: formatDuration(lengthSeconds),
+                    language: ytpSegments[0]?.lang ?? preferredLang,
+                    segments,
+                    fullText,
+                    availableLanguages,
+                    description,
+                    publishDate,
+                    chapters: chapters.length > 0 ? chapters : undefined,
+                    keyPoints: keyPoints.length > 0 ? keyPoints : undefined,
+                    summary,
+                    wordCount,
+                };
+            }
+            console.log('[webpeel] [youtube] Path 0 returned empty segments');
+        }
+        catch (err) {
+            console.log('[webpeel] [youtube] Path 0 failed:', err?.message);
+        }
+    } // end VITEST guard
     const ytUserAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
     const ytHeaders = {
         'Cookie': 'SOCS=CAISNQgDEitib3FfaWRlbnRpdHlmcm9udGVuZHVpc2VydmVyXzIwMjQwNTE1LjA3X3AxGgJlbiADGgYIgLv3tQY; CONSENT=PENDING+987',
         'Accept-Language': 'en-US,en;q=0.9',
     };
+    // --- Path 1: yt-dlp approach (most reliable on cloud servers — handles signature challenges internally) ---
+    if (ytdlpAvailable) {
+        console.log('[webpeel] [youtube] Trying path 1: yt-dlp');
+        try {
+            const ytdlpResult = await getTranscriptViaYtDlp(videoId, preferredLang);
+            if (ytdlpResult && ytdlpResult.segments.length > 0) {
+                return ytdlpResult;
+            }
+            console.log('[webpeel] [youtube] Path 1 failed: yt-dlp returned no segments');
+        }
+        catch (err) {
+            console.log('[webpeel] [youtube] Path 1 failed:', err?.message);
+        }
+    }
+    else {
+        console.log('[webpeel] [youtube] Skipping path 1: yt-dlp not available');
+    }
+    // --- Path 2: HTTP fetch (simpleFetch first; if our challenge detection fires, fall back to native fetch) ---
+    // YouTube serves consent/challenge pages to server IPs without cookies.
+    // Setting SOCS consent cookie bypasses this — same approach as youtube-transcript npm.
+    // On cloud servers, simpleFetch may throw BlockedError due to our own challenge detection;
+    // in that case we retry with native fetch() which bypasses that guard.
+    console.log('[webpeel] [youtube] Trying path 2: native fetch');
     try {
-        const fetchResult = await simpleFetch(videoUrl, ytUserAgent, 15000, ytHeaders);
-        const html = fetchResult.html;
+        let html;
+        try {
+            const fetchResult = await simpleFetch(videoUrl, ytUserAgent, 15000, ytHeaders);
+            html = fetchResult.html;
+        }
+        catch (simpleFetchErr) {
+            // If our own challenge detection threw BlockedError, retry with raw native fetch
+            const errMsg = (simpleFetchErr?.message ?? '').toLowerCase();
+            const isBlocked = simpleFetchErr?.constructor?.name === 'BlockedError' ||
+                errMsg.includes('blocked') ||
+                errMsg.includes('challenge') ||
+                errMsg.includes('cloudflare');
+            if (!isBlocked)
+                throw simpleFetchErr;
+            console.log('[webpeel] [youtube] simpleFetch BlockedError — retrying with native fetch');
+            const fetchResponse = await fetch(videoUrl, {
+                headers: {
+                    'User-Agent': ytUserAgent,
+                    ...ytHeaders,
+                },
+                redirect: 'follow',
+                signal: AbortSignal.timeout(15000),
+            });
+            html = await fetchResponse.text();
+        }
         if (!html.includes('ytInitialPlayerResponse') && !html.includes('ytInitialData')) {
             throw new Error('YouTube served non-video page (likely challenge/consent)');
         }
@@ -257,7 +626,7 @@ export async function getYouTubeTranscript(url, options = {}) {
         const segments = parseCaptionXml(captionXml);
         if (segments.length === 0) {
             // Caption URL returned empty content (common when ip=0.0.0.0 in signature)
-            // Fall through to yt-dlp / browser intercept path
+            // Fall through to browser intercept path
             throw new Error('Caption XML returned empty — session-locked URL');
         }
         const fullText = segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim();
@@ -288,23 +657,14 @@ export async function getYouTubeTranscript(url, options = {}) {
         if (msg.includes('No captions available') || msg.includes('Not a valid YouTube URL')) {
             throw err;
         }
+        console.log('[webpeel] [youtube] Path 2 failed:', msg);
         // Network/parsing failures — fall through to browser intercept approach
     }
-    // --- Path 2: yt-dlp approach (fast, reliable, handles signature challenges) ---
-    try {
-        const ytdlpResult = await getTranscriptViaYtDlp(videoId, preferredLang);
-        if (ytdlpResult && ytdlpResult.segments.length > 0) {
-            return ytdlpResult;
-        }
-    }
-    catch (err) {
-        if (process.env.DEBUG)
-            console.debug('[webpeel]', 'yt-dlp transcript failed:', err?.message);
-    }
     // --- Path 3: Browser intercept approach ---
     // YouTube's caption URLs are session-specific (they return empty when fetched
     // from a different HTTP client). We intercept the timedtext network request
     // that the YouTube player makes automatically when loading the page.
+    console.log('[webpeel] [youtube] Trying path 3: browser intercept');
     return getTranscriptViaBrowserIntercept(videoId, videoUrl, preferredLang);
 }
 /**
@@ -333,7 +693,7 @@ async function getTranscriptViaYtDlp(videoId, preferredLang) {
             ...process.env,
             PATH: `/usr/local/bin:/usr/bin:/bin:${process.env.PATH ?? ''}`,
         };
-        const proc = execFile('yt-dlp', args, { timeout: 45000, env: execEnv }, async (err) => {
+        const proc = execFile('yt-dlp', args, { timeout: 60000, env: execEnv }, async (err) => {
             try {
                 if (err) {
                     // yt-dlp not installed, timed out, or failed
@@ -687,10 +1047,35 @@ function selectBestTrack(tracks, preferredLang) {
 /**
  * Fetch the caption XML from YouTube's timedtext API.
  * Must use same cookies/UA as the page fetch — URLs are session-locked.
+ * Tries simpleFetch first; falls back to native fetch() if BlockedError is thrown
+ * (our own challenge detection fires on cloud server IPs).
  */
 async function fetchCaptionXml(baseUrl, userAgent, headers) {
-    const result = await simpleFetch(baseUrl, userAgent, 10000, headers);
-    return result.html;
+    try {
+        const result = await simpleFetch(baseUrl, userAgent, 10000, headers);
+        return result.html;
+    }
+    catch (simpleFetchErr) {
+        const errMsg = (simpleFetchErr?.message ?? '').toLowerCase();
+        const isBlocked = simpleFetchErr?.constructor?.name === 'BlockedError' ||
+            errMsg.includes('blocked') ||
+            errMsg.includes('challenge') ||
+            errMsg.includes('cloudflare');
+        if (!isBlocked)
+            throw simpleFetchErr;
+        // BlockedError: retry with native fetch
+        const fetchHeaders = {};
+        if (userAgent)
+            fetchHeaders['User-Agent'] = userAgent;
+        if (headers)
+            Object.assign(fetchHeaders, headers);
+        const response = await fetch(baseUrl, {
+            headers: fetchHeaders,
+            redirect: 'follow',
+            signal: AbortSignal.timeout(10000),
+        });
+        return response.text();
+    }
 }
 /**
  * Parse YouTube caption XML into transcript segments.

package/dist/server/app.js

@@ -104,7 +104,7 @@ export function createApp(config = {}) {
         else if (req.query?.render === 'true')
             timeoutMs = 60000; // 1min for rendered fetches
         else if (urlParam.includes('youtube.com') || urlParam.includes('youtu.be'))
-            timeoutMs = 45000; // 45s for YouTube (transcript extraction)
+            timeoutMs = 90000; // 90s for YouTube (yt-dlp needs time after simpleFetch fails)
         req.setTimeout(timeoutMs);
         res.setTimeout(timeoutMs, () => {
             if (!res.headersSent) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.20.13",
+  "version": "0.20.17",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",
@@ -112,7 +112,9 @@
     "resend": "^6.9.3",
     "turndown": "^7.2.0",
     "turndown-plugin-gfm": "^1.0.2",
-    "undici": "^7.2.0"
+    "undici": "^7.2.0",
+    "youtube-transcript": "^1.2.1",
+    "youtube-transcript-plus": "^1.2.0"
   },
   "optionalDependencies": {
     "@sentry/node": "^7.120.4",