webpeel 0.20.5 → 0.20.7

package/dist/cli/commands/fetch.js

@@ -7,9 +7,40 @@ import { getProfilePath, loadStorageState, touchProfile } from '../../core/profi
 import { peel, cleanup } from '../../index.js';
 import { checkUsage, showUsageFooter, loadConfig } from '../../cli-auth.js';
 import { getCache, setCache, parseTTL } from '../../cache.js';
-import { estimateTokens } from '../../core/markdown.js';
+import { estimateTokens, htmlToMarkdown } from '../../core/markdown.js';
 import { distillToBudget, budgetListings } from '../../core/budget.js';
 import { parseActions, formatError, fetchViaApi, outputResult, writeStdout, buildEnvelope, classifyErrorCode, formatListingsCsv, normaliseExtractedToRows, } from '../utils.js';
+// ─── readStdin ────────────────────────────────────────────────────────────────
+async function readStdin() {
+    const chunks = [];
+    for await (const chunk of process.stdin) {
+        chunks.push(Buffer.from(chunk));
+    }
+    return Buffer.concat(chunks).toString('utf-8');
+}
+// ─── runStdin ─────────────────────────────────────────────────────────────────
+// Read HTML from stdin, convert to markdown, and output
+async function runStdin(options) {
+    try {
+        const html = await readStdin();
+        if (!html.trim()) {
+            process.stderr.write('Error: No input received on stdin\n');
+            process.exit(1);
+        }
+        const markdown = htmlToMarkdown(html, { raw: false, prune: true });
+        if (options.json) {
+            const tokens = estimateTokens(markdown);
+            process.stdout.write(JSON.stringify({ success: true, content: markdown, tokens }) + '\n');
+        }
+        else {
+            process.stdout.write(markdown + '\n');
+        }
+    }
+    catch (err) {
+        process.stderr.write(`Error: ${err.message}\n`);
+        process.exit(1);
+    }
+}
 // ─── runFetch ─────────────────────────────────────────────────────────────────
 // Main fetch handler — shared with the `pipe` and `ask` subcommands
 export async function runFetch(url, options) {
@@ -1045,7 +1076,12 @@ export function registerFetchCommands(program) {
         .option('--format <type>', 'Output format: markdown (default), text, html, json')
         .option('--content-only', 'Output only the raw content field (no metadata, no JSON wrapper) — ideal for piping to LLMs')
         .option('--progress', 'Show engine escalation steps (simple → browser → stealth) with timing')
+        .option('--stdin', 'Read HTML from stdin instead of fetching a URL — converts to markdown')
         .action(async (url, options) => {
+        if (options.stdin) {
+            await runStdin(options);
+            return;
+        }
         await runFetch(url, options);
     });
     // ── read subcommand (explicit readable mode) ─────────────────────────────

package/dist/core/domain-extractors.js

@@ -324,19 +324,87 @@ async function twitterExtractor(html, url) {
                     source: 'fxtwitter',
                 };
                 // Try to fetch recent tweets from Twitter's public syndication endpoint
+                // NOTE: simpleFetch sends too many Sec-* headers that trigger 429. Use https directly.
                 let recentTweets = '';
                 try {
-                    const syndicationUrl = `https://syndication.twitter.com/srv/timeline-profile/screen-name/${u.screen_name}`;
-                    const syndicationResult = await simpleFetch(syndicationUrl, 'Mozilla/5.0 (compatible; WebPeel/1.0)', 8000);
-                    if (syndicationResult?.html) {
-                        // Extract tweet texts from the syndication HTML
-                        const tweetMatches = [...syndicationResult.html.matchAll(/"full_text":"((?:[^"\\]|\\.)*)"/g)];
-                        const tweets = tweetMatches
-                            .slice(0, 5)
-                            .map(m => m[1].replace(/\\n/g, ' ').replace(/\\"/g, '"').trim())
-                            .filter(t => t.length > 10 && !t.startsWith('RT @'));
-                        if (tweets.length > 0) {
-                            recentTweets = '\n\n### Recent Tweets\n\n' + tweets.map(t => `> ${t}`).join('\n\n');
+                    const { default: httpsModule } = await import('https');
+                    const syndicationHtml = await new Promise((resolve, reject) => {
+                        const req = httpsModule.request({
+                            hostname: 'syndication.twitter.com',
+                            path: `/srv/timeline-profile/screen-name/${u.screen_name}`,
+                            method: 'GET',
+                            headers: {
+                                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                                'Accept-Language': 'en-US,en;q=0.9',
+                            },
+                        }, (res) => {
+                            if (res.statusCode !== 200) {
+                                reject(new Error(`HTTP ${res.statusCode}`));
+                                res.resume();
+                                return;
+                            }
+                            let body = '';
+                            res.on('data', (chunk) => body += chunk.toString());
+                            res.on('end', () => resolve(body));
+                        });
+                        req.on('error', reject);
+                        setTimeout(() => req.destroy(new Error('timeout')), 12000);
+                        req.end();
+                    });
+                    if (syndicationHtml) {
+                        // Parse __NEXT_DATA__ JSON from the syndication page for rich tweet data
+                        const nextDataMatch = syndicationHtml.match(/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/i);
+                        if (nextDataMatch) {
+                            const nextData = tryParseJson(nextDataMatch[1]);
+                            const entries = nextData?.props?.pageProps?.timeline?.entries || [];
+                            const tweetSections = [];
+                            for (const entry of entries) {
+                                if (tweetSections.length >= 8)
+                                    break;
+                                const tweet = entry?.content?.tweet;
+                                if (!tweet?.full_text)
+                                    continue;
+                                const text = tweet.full_text.replace(/\\n/g, '\n').replace(/\\"/g, '"').trim();
+                                // Skip retweets and pure-URL-only tweets without media
+                                if (text.startsWith('RT @'))
+                                    continue;
+                                const media = tweet.extended_entities?.media || tweet.entities?.media || [];
+                                const isUrlOnly = /^https?:\/\/t\.co\/\S+$/.test(text.trim()) || /^https?:\/\/t\.co\/\S+\s*$/.test(text.trim());
+                                if (isUrlOnly && media.length === 0)
+                                    continue;
+                                // Format date
+                                const dateStr = tweet.created_at ? (() => {
+                                    try {
+                                        return new Date(tweet.created_at).toLocaleDateString('en-US', { year: 'numeric', month: 'long', day: 'numeric' });
+                                    }
+                                    catch {
+                                        return tweet.created_at;
+                                    }
+                                })() : '';
+                                const likes = tweet.favorite_count ?? 0;
+                                const retweets = tweet.retweet_count ?? 0;
+                                const replies = tweet.reply_count ?? 0;
+                                const fmtNum = (n) => n >= 1000000 ? (n / 1000000).toFixed(1) + 'M' : n >= 1000 ? (n / 1000).toFixed(1) + 'K' : String(n);
+                                const mediaLine = media.length > 0 ? `\n📷 ${media.map((m) => m.media_url_https || m.media_url).filter(Boolean).join(', ')}` : '';
+                                // Clean t.co URLs from text when they have real media
+                                const cleanText = media.length > 0 ? text.replace(/https?:\/\/t\.co\/\S+/g, '').trim() : text;
+                                tweetSections.push(`### ${dateStr}\n${cleanText}${mediaLine}\n♻️ ${fmtNum(retweets)} | ❤️ ${fmtNum(likes)} | 💬 ${fmtNum(replies)}`);
+                            }
+                            if (tweetSections.length > 0) {
+                                recentTweets = '\n\n## Recent Tweets\n\n' + tweetSections.join('\n\n---\n\n');
+                            }
+                        }
+                        else {
+                            // Fallback: simple regex extraction without metrics
+                            const tweetMatches = [...syndicationHtml.matchAll(/"full_text":"((?:[^"\\]|\\.)*)"/g)];
+                            const tweets = tweetMatches
+                                .slice(0, 5)
+                                .map(m => m[1].replace(/\\n/g, ' ').replace(/\\"/g, '"').trim())
+                                .filter(t => t.length > 10 && !t.startsWith('RT @'));
+                            if (tweets.length > 0) {
+                                recentTweets = '\n\n## Recent Tweets\n\n' + tweets.map(t => `> ${t}`).join('\n\n');
+                            }
                         }
                     }
                 }
@@ -344,7 +412,7 @@ async function twitterExtractor(html, url) {
                 const websiteLine = structured.website ? `\n🌐 ${structured.website}` : '';
                 const joinedLine = structured.created ? `\n📅 Joined: ${structured.created}` : '';
                 const likesLine = structured.likes ? `  |  ❤️ Likes: ${structured.likes?.toLocaleString() || 0}` : '';
-                const cleanContent = `## 🐦 @${(structured.handle || '').replace('@', '')} on X/Twitter\n\n**${structured.name}**${structured.verified ? ' ✓' : ''}\n\n${structured.bio || ''}\n\n📍 ${structured.location || 'N/A'}${websiteLine}${joinedLine}\n\n👥 **Followers:** ${structured.followers?.toLocaleString() || 0}  |  **Following:** ${structured.following?.toLocaleString() || 0}  |  **Tweets:** ${structured.tweets?.toLocaleString() || 0}${likesLine}${recentTweets}`;
+                const cleanContent = `# @${(structured.handle || '').replace('@', '')} on X/Twitter\n\n**${structured.name}**${structured.verified ? ' ✓' : ''}\n\n${structured.bio || ''}\n\n📍 ${structured.location || 'N/A'}${websiteLine}${joinedLine}\n👥 Followers: ${structured.followers?.toLocaleString() || 0} | Following: ${structured.following?.toLocaleString() || 0} | Tweets: ${structured.tweets?.toLocaleString() || 0}${likesLine}${recentTweets}`;
                 return { domain, type: 'profile', structured, cleanContent };
             }
         }
@@ -1961,7 +2029,16 @@ async function linkedinExtractor(html, url) {
     try {
         const { load } = await import('cheerio');
         const $ = load(html);
-        // LinkedIn SSR exposes some data in meta tags and JSON-LD
+        // Detect page type from URL first
+        const urlObj = new URL(url);
+        const pathParts = urlObj.pathname.split('/').filter(Boolean);
+        const pageType = pathParts[0] === 'company' ? 'company'
+            : pathParts[0] === 'in' ? 'profile'
+                : pathParts[0] === 'jobs' ? 'job'
+                    : 'page';
+        // Detect if we're on the authwall (LinkedIn redirects unauthenticated requests)
+        const isAuthwall = html.includes('authwall') || html.includes('Join LinkedIn') || html.includes('Sign in') && !html.includes('linkedin.com/in/');
+        // --- Try parsing meta tags / JSON-LD from the HTML ---
         let jsonLd = null;
         $('script[type="application/ld+json"]').each((_, el) => {
             if (jsonLd)
@@ -1974,30 +2051,83 @@ async function linkedinExtractor(html, url) {
         const ogTitle = $('meta[property="og:title"]').attr('content') || '';
         const ogDescription = $('meta[property="og:description"]').attr('content') || '';
         const ogImage = $('meta[property="og:image"]').attr('content') || '';
-        const name = jsonLd?.name || ogTitle.replace(/ \| LinkedIn$/, '').trim() || '';
+        const metaDescription = $('meta[name="description"]').attr('content') || '';
+        let name = jsonLd?.name || ogTitle.replace(/ \| LinkedIn$/, '').replace(/Sign Up \| LinkedIn$/, '').trim() || '';
+        // When on authwall, discard authwall-specific meta data
+        let headline = isAuthwall ? (jsonLd?.jobTitle || '') : (jsonLd?.jobTitle || metaDescription?.split('|')?.[0]?.trim() || ogDescription || '');
+        let description = isAuthwall ? (jsonLd?.description || '') : (jsonLd?.description || ogDescription || '');
+        let location = $('[class*="location"]').first().text().trim() || jsonLd?.address?.addressLocality || '';
+        // --- If authwall or no useful data, try direct HTTPS fetch with minimal headers ---
+        // LinkedIn returns rich og: meta tags when fetched with a plain browser UA (no Sec-Fetch-* noise)
+        if (!name || isAuthwall || name.toLowerCase().includes('sign up') || name.toLowerCase().includes('linkedin')) {
+            try {
+                const { default: httpsLI } = await import('https');
+                const { gunzip } = await import('zlib');
+                const linkedInHtml = await new Promise((resolve, reject) => {
+                    const req = httpsLI.request({
+                        hostname: 'www.linkedin.com',
+                        path: urlObj.pathname,
+                        method: 'GET',
+                        headers: {
+                            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                            'Accept': 'text/html,application/xhtml+xml',
+                            'Accept-Language': 'en-US,en;q=0.9',
+                            'Accept-Encoding': 'gzip, deflate',
+                        },
+                    }, (res) => {
+                        if (res.statusCode && res.statusCode >= 400) {
+                            reject(new Error(`HTTP ${res.statusCode}`));
+                            res.resume();
+                            return;
+                        }
+                        const chunks = [];
+                        res.on('data', (chunk) => chunks.push(chunk));
+                        res.on('end', () => {
+                            const buf = Buffer.concat(chunks);
+                            const enc = res.headers['content-encoding'] || '';
+                            if (enc === 'gzip') {
+                                gunzip(buf, (err, decoded) => err ? reject(err) : resolve(decoded.toString('utf8')));
+                            }
+                            else {
+                                resolve(buf.toString('utf8'));
+                            }
+                        });
+                    });
+                    req.on('error', reject);
+                    setTimeout(() => req.destroy(new Error('timeout')), 10000);
+                    req.end();
+                });
+                if (linkedInHtml) {
+                    const $li = load(linkedInHtml);
+                    const liOgTitle = $li('meta[property="og:title"]').attr('content') || '';
+                    const liOgDesc = $li('meta[property="og:description"]').attr('content') || '';
+                    // Only use if it has real profile data (not authwall)
+                    if (liOgTitle && !liOgTitle.toLowerCase().includes('sign up') && !liOgTitle.toLowerCase().includes('join linkedin')) {
+                        // "Name - Headline | LinkedIn" or "Name | LinkedIn"
+                        const titleParts = liOgTitle.replace(/ \| LinkedIn$/, '').split(/\s*[-–]\s*/);
+                        if (titleParts[0])
+                            name = titleParts[0].trim();
+                        if (titleParts[1])
+                            headline = titleParts[1].trim();
+                        if (liOgDesc)
+                            description = liOgDesc;
+                    }
+                }
+            }
+            catch { /* direct fetch optional */ }
+        }
         if (!name)
             return null;
-        const headline = jsonLd?.jobTitle ||
-            $('meta[name="description"]').attr('content')?.split('|')?.[0]?.trim() ||
-            ogDescription || '';
-        const description = jsonLd?.description || ogDescription || '';
-        // Try to detect page type from URL
-        const pathParts = new URL(url).pathname.split('/').filter(Boolean);
-        const pageType = pathParts[0] === 'company' ? 'company'
-            : pathParts[0] === 'in' ? 'profile'
-                : pathParts[0] === 'jobs' ? 'job'
-                    : 'page';
-        // Extract any visible structured info from the HTML
-        const location = $('[class*="location"]').first().text().trim() ||
-            jsonLd?.address?.addressLocality || '';
         const structured = {
             name, headline, description, location, pageType,
             image: ogImage, url,
         };
         const typeLine = pageType === 'company' ? '🏢' : pageType === 'profile' ? '👤' : '🔗';
         const locationLine = location ? `\n📍 ${location}` : '';
-        const headlineLine = headline ? `\n*${headline}*` : '';
-        const cleanContent = `# ${typeLine} ${name}${headlineLine}${locationLine}\n\n${description}`;
+        const headlineLine = headline && headline !== name ? `\n*${headline}*` : '';
+        const descriptionLine = description ? `\n\n${description}` : '';
+        const authNote = '\n\n⚠️ Full LinkedIn profiles require authentication. Use /v1/session to log in first.';
+        const cleanContent = `# ${typeLine} ${name} — LinkedIn${headlineLine}${locationLine}${descriptionLine}${authNote}`;
         return { domain: 'linkedin.com', type: pageType, structured, cleanContent };
     }
     catch {
@@ -2541,8 +2671,73 @@ async function soundcloudExtractor(_html, url) {
 // 29. Instagram extractor (oEmbed)
 // ---------------------------------------------------------------------------
 async function instagramExtractor(_html, url) {
+    const pathParts = new URL(url).pathname.split('/').filter(Boolean);
+    const contentType = pathParts[0] === 'p' ? 'post' : pathParts[0] === 'reel' ? 'reel' : pathParts[0] === 'tv' ? 'igtv' : pathParts.length === 1 ? 'profile' : 'post';
+    // --- Profile extraction via Instagram internal API (no auth needed) ---
+    if (contentType === 'profile' && pathParts.length === 1) {
+        const username = pathParts[0];
+        try {
+            const apiUrl = `https://www.instagram.com/api/v1/users/web_profile_info/?username=${encodeURIComponent(username)}`;
+            const igHeaders = {
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                'X-IG-App-ID': '936619743392459',
+                'Accept': '*/*',
+                'Referer': 'https://www.instagram.com/',
+            };
+            const apiResult = await simpleFetch(apiUrl, igHeaders['User-Agent'], 12000, igHeaders);
+            const data = tryParseJson(apiResult?.html || '');
+            const user = data?.data?.user;
+            if (user && user.username) {
+                const followers = user.edge_followed_by?.count ?? 0;
+                const following = user.edge_follow?.count ?? 0;
+                const postCount = user.edge_owner_to_timeline_media?.count ?? 0;
+                const fmtNum = (n) => n >= 1000000 ? (n / 1000000).toFixed(1) + 'M' : n >= 1000 ? (n / 1000).toFixed(1) + 'K' : String(n);
+                const structured = {
+                    username: user.username,
+                    fullName: user.full_name || '',
+                    bio: user.biography || '',
+                    followers,
+                    following,
+                    posts: postCount,
+                    verified: user.is_verified || false,
+                    isPrivate: user.is_private || false,
+                    profilePic: user.profile_pic_url_hd || user.profile_pic_url || '',
+                    externalUrl: user.external_url || (user.bio_links?.[0]?.url) || '',
+                    contentType: 'profile',
+                };
+                // Recent posts
+                const edges = user.edge_owner_to_timeline_media?.edges || [];
+                const postSections = [];
+                for (const edge of edges.slice(0, 6)) {
+                    const node = edge?.node;
+                    if (!node)
+                        continue;
+                    const caption = node.edge_media_to_caption?.edges?.[0]?.node?.text || '';
+                    const likes = node.edge_liked_by?.count ?? node.edge_media_preview_like?.count ?? 0;
+                    const comments = node.edge_media_to_comment?.count ?? 0;
+                    const isVideo = node.is_video;
+                    const mediaType = isVideo ? '🎬' : '📸';
+                    const timestamp = node.taken_at_timestamp ? new Date(node.taken_at_timestamp * 1000).toLocaleDateString('en-US', { year: 'numeric', month: 'long', day: 'numeric' }) : '';
+                    const imgUrl = node.thumbnail_src || node.display_url || '';
+                    const captionSnippet = caption ? caption.slice(0, 150) + (caption.length > 150 ? '…' : '') : '';
+                    postSections.push(`### ${mediaType} ${timestamp}\n${captionSnippet}\n❤️ ${fmtNum(likes)} | 💬 ${fmtNum(comments)}${imgUrl ? `\n🖼 ${imgUrl}` : ''}`);
+                }
+                const verifiedBadge = structured.verified ? ' ✓' : '';
+                const privateBadge = structured.isPrivate ? ' 🔒' : '';
+                const bioLine = structured.bio ? `\n\n${structured.bio}` : '';
+                const externalLine = structured.externalUrl ? `\n🌐 ${structured.externalUrl}` : '';
+                const postsSection = postSections.length > 0 ? '\n\n## Recent Posts\n\n' + postSections.join('\n\n---\n\n') : '';
+                const cleanContent = `# @${structured.username} on Instagram${verifiedBadge}${privateBadge}\n\n**${structured.fullName || structured.username}**${bioLine}${externalLine}\n\n👥 ${fmtNum(followers)} Followers | ${fmtNum(following)} Following | ${fmtNum(postCount)} Posts${postsSection}`;
+                return { domain: 'instagram.com', type: 'profile', structured, cleanContent };
+            }
+        }
+        catch (e) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'Instagram profile API failed:', e instanceof Error ? e.message : e);
+        }
+    }
+    // --- Post/Reel/IGTV: Try oEmbed API ---
     try {
-        // Instagram official oEmbed (no access token needed for basic data)
         const oembedUrl = `https://graph.facebook.com/v22.0/instagram_oembed?url=${encodeURIComponent(url)}&fields=title,author_name,provider_name,thumbnail_url`;
         const data = await fetchJson(oembedUrl);
         // Also try noembed.com as fallback
@@ -2553,8 +2748,6 @@ async function instagramExtractor(_html, url) {
         }
         if (!resolvedData || resolvedData.error)
             return null;
-        const pathParts = new URL(url).pathname.split('/').filter(Boolean);
-        const contentType = pathParts[0] === 'p' ? 'post' : pathParts[0] === 'reel' ? 'reel' : pathParts[0] === 'tv' ? 'igtv' : pathParts.length === 1 ? 'profile' : 'post';
         const structured = {
             title: resolvedData.title || '',
             author: resolvedData.author_name || '',
@@ -2563,7 +2756,7 @@ async function instagramExtractor(_html, url) {
             contentType,
             provider: 'Instagram',
         };
-        const typeEmoji = contentType === 'reel' ? '🎬' : contentType === 'post' ? '📸' : contentType === 'profile' ? '👤' : '📱';
+        const typeEmoji = contentType === 'reel' ? '🎬' : contentType === 'post' ? '📸' : '📱';
         const titleText = structured.title || `Instagram ${contentType} by ${structured.author}`;
         const cleanContent = `## ${typeEmoji} Instagram ${contentType}: ${titleText}\n\n**Creator:** @${structured.author.replace('@', '')}\n**URL:** ${url}`;
         return { domain: 'instagram.com', type: contentType, structured, cleanContent };
@@ -2575,59 +2768,106 @@ async function instagramExtractor(_html, url) {
     }
 }
 // ---------------------------------------------------------------------------
-// 30. PDF extractor (URL-based detection)
+// 30. PDF extractor (URL-based detection) — downloads and extracts real text
 // ---------------------------------------------------------------------------
+const PDF_MAX_BYTES = 50 * 1024 * 1024; // 50 MB
+const PDF_TRUNCATE_CHARS = 100_000;
 async function pdfExtractor(_html, url) {
     try {
         const urlObj = new URL(url);
         const filename = urlObj.pathname.split('/').pop() || 'document.pdf';
         const hostname = urlObj.hostname;
-        // Try to get HEAD request metadata
-        let contentType = 'application/pdf';
-        let contentLength = '';
+        // Download the PDF
+        let buffer;
+        let finalContentType = 'application/pdf';
         try {
-            const { default: https } = await import('https');
-            const { default: http } = await import('http');
-            const client = url.startsWith('https') ? https : http;
-            await new Promise((resolve) => {
-                const req = client.request(url, { method: 'HEAD', timeout: 5000 }, (res) => {
-                    contentType = res.headers['content-type'] || 'application/pdf';
-                    contentLength = res.headers['content-length'] || '';
-                    resolve();
-                    res.resume();
-                });
-                req.on('error', () => resolve());
-                req.on('timeout', () => { req.destroy(); resolve(); });
-                req.end();
+            const response = await fetch(url, {
+                headers: { 'User-Agent': 'Mozilla/5.0 (compatible; WebPeel/1.0)' },
+                signal: AbortSignal.timeout(30000),
             });
+            if (!response.ok) {
+                if (process.env.DEBUG)
+                    console.debug('[webpeel]', `PDF download failed: HTTP ${response.status}`);
+                return null; // Let the normal pipeline handle it
+            }
+            finalContentType = response.headers.get('content-type') || 'application/pdf';
+            // Verify it's actually a PDF (content-type or URL)
+            const isPdf = finalContentType.toLowerCase().includes('pdf') || /\.pdf(\?|$|#)/i.test(url);
+            if (!isPdf)
+                return null;
+            const arrayBuffer = await response.arrayBuffer();
+            buffer = Buffer.from(arrayBuffer);
+        }
+        catch (downloadErr) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'PDF download error:', downloadErr instanceof Error ? downloadErr.message : downloadErr);
+            return null; // Let the normal pipeline handle it
         }
-        catch { /* best-effort */ }
-        const isPdf = contentType.toLowerCase().includes('pdf') || /\.pdf(\?|$|#)/i.test(url);
-        if (!isPdf)
+        // Size guard
+        if (buffer.length > PDF_MAX_BYTES) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', `PDF too large (${buffer.length} bytes), falling back to stub`);
             return null;
-        const fileSizeKb = contentLength ? Math.round(parseInt(contentLength) / 1024) : null;
-        const structured = {
-            filename,
-            url,
-            contentType,
-            fileSizeKb,
-            hostname,
-            note: 'PDF binary content — text extraction requires a PDF parser. Use a dedicated PDF extraction service for full text content.',
+        }
+        // Extract text via pdf-parse
+        const { extractPdf } = await import('./pdf.js');
+        let pdf;
+        try {
+            pdf = await extractPdf(buffer);
+        }
+        catch (parseErr) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'PDF parse failed:', parseErr instanceof Error ? parseErr.message : parseErr);
+            return null; // Let the normal pipeline handle it
+        }
+        // Normalize whitespace (pdf-parse emits lots of blank lines)
+        let text = (pdf.text || '')
+            .replace(/\r\n/g, '\n')
+            .replace(/\n{3,}/g, '\n\n')
+            .replace(/[ \t]+/g, ' ')
+            .trim();
+        // Truncate very large documents
+        let truncated = false;
+        if (text.length > PDF_TRUNCATE_CHARS) {
+            text = text.slice(0, PDF_TRUNCATE_CHARS);
+            truncated = true;
+        }
+        if (!text) {
+            // Scanned/image-only PDF — return a clear message rather than empty content
+            const emptyNote = `## 📄 ${filename}\n\n*This PDF appears to be a scanned document (image-only). No extractable text was found.*\n\n**Source:** ${url}`;
+            return {
+                domain: hostname,
+                type: 'pdf',
+                structured: { title: filename, url, pages: pdf.pages, contentType: finalContentType },
+                cleanContent: emptyNote,
+            };
+        }
+        // Build markdown output
+        const titleRaw = pdf.metadata?.title || '';
+        const title = titleRaw || filename.replace(/\.pdf$/i, '') || 'PDF Document';
+        const metaParts = [];
+        if (pdf.metadata?.author)
+            metaParts.push(`**Author:** ${pdf.metadata.author}`);
+        if (pdf.pages)
+            metaParts.push(`**Pages:** ${pdf.pages}`);
+        metaParts.push(`**Source:** ${url}`);
+        const header = titleRaw ? `# ${titleRaw}\n\n` : '';
+        const metaBlock = metaParts.join(' | ') + '\n\n';
+        const truncNote = truncated ? '\n\n*[Content truncated — document exceeds 100,000 characters]*' : '';
+        const cleanContent = header + metaBlock + text + truncNote;
+        return {
+            domain: hostname,
+            type: 'pdf',
+            structured: {
+                title,
+                filename,
+                url,
+                pages: pdf.pages,
+                contentType: finalContentType,
+                ...pdf.metadata,
+            },
+            cleanContent,
         };
-        const sizeStr = fileSizeKb ? ` (${fileSizeKb > 1024 ? (fileSizeKb / 1024).toFixed(1) + ' MB' : fileSizeKb + ' KB'})` : '';
-        const cleanContent = `## 📄 PDF Document: ${filename}
-
-**URL:** ${url}
-**Host:** ${hostname}${sizeStr ? `\n**Size:** ${sizeStr}` : ''}
-
-> **Note:** This is a PDF document. Binary PDF content cannot be directly extracted as text through standard web fetching. To extract the full text, consider:
->
-> 1. Use a dedicated PDF extraction service (e.g., Adobe PDF Extract API, pdfminer, PyMuPDF)
-> 2. Download the file and process locally with \`pdf-parse\` (Node.js) or \`pdfplumber\` (Python)
-> 3. For academic PDFs, check if an HTML version is available at the same URL without \`.pdf\`
-
-**Direct download URL:** ${url}`;
-        return { domain: hostname, type: 'pdf', structured, cleanContent };
     }
     catch (e) {
         if (process.env.DEBUG)

package/dist/server/routes/agent.d.ts

@@ -1,5 +1,7 @@
 /**
- * POST /v1/agent
+ * POST /v1/agent           — single autonomous agent query
+ * POST /v1/agent/batch     — parallel batch of agent queries (max 50)
+ * GET  /v1/agent/batch/:id — poll batch job status
  *
  * Autonomous web agent — search → fetch → extract (LLM or BM25)
  *
@@ -11,9 +13,7 @@
  *
  * Returns: { success, data|answer, sources, method, elapsed, tokensUsed }
  *
- * Two modes:
- *   - agent-llm:  schema + llmApiKey → LLM extraction (BYOK)
- *   - agent-bm25: no LLM key → BM25 text answer (always free)
+ * Webhook support: pass `webhook` URL to get async delivery with HMAC-SHA256 signing.
  *
  * 5-minute in-memory cache. Max 10 sources per request.
  */

package/dist/server/routes/agent.js

@@ -1,5 +1,7 @@
 /**
- * POST /v1/agent
+ * POST /v1/agent           — single autonomous agent query
+ * POST /v1/agent/batch     — parallel batch of agent queries (max 50)
+ * GET  /v1/agent/batch/:id — poll batch job status
  *
  * Autonomous web agent — search → fetch → extract (LLM or BM25)
  *
@@ -11,9 +13,7 @@
  *
  * Returns: { success, data|answer, sources, method, elapsed, tokensUsed }
  *
- * Two modes:
- *   - agent-llm:  schema + llmApiKey → LLM extraction (BYOK)
- *   - agent-bm25: no LLM key → BM25 text answer (always free)
+ * Webhook support: pass `webhook` URL to get async delivery with HMAC-SHA256 signing.
  *
  * 5-minute in-memory cache. Max 10 sources per request.
  */
@@ -22,9 +22,42 @@ import { peel } from '../../index.js';
 import { extractWithLLM } from '../../core/llm-extract.js';
 import { getBestSearchProvider } from '../../core/search-provider.js';
 import { quickAnswer } from '../../core/quick-answer.js';
+import { sendWebhook } from './webhooks.js';
 import { createLogger } from '../../core/logger.js';
 import crypto from 'crypto';
 const log = createLogger('agent');
+const batchJobs = new Map();
+const BATCH_TTL = 60 * 60 * 1000; // 1 hour
+// GC stale batch jobs every 10 minutes
+setInterval(() => {
+    const now = Date.now();
+    for (const [id, job] of batchJobs) {
+        if (now - job.createdAt > BATCH_TTL)
+            batchJobs.delete(id);
+    }
+}, 10 * 60 * 1000).unref();
+// Simple concurrency limiter
+class Semaphore {
+    max;
+    queue = [];
+    running = 0;
+    constructor(max) {
+        this.max = max;
+    }
+    async acquire() {
+        if (this.running < this.max) {
+            this.running++;
+            return;
+        }
+        return new Promise((resolve) => this.queue.push(() => { this.running++; resolve(); }));
+    }
+    release() {
+        this.running--;
+        const next = this.queue.shift();
+        if (next)
+            next();
+    }
+}
 const cache = new Map();
 const CACHE_TTL = 5 * 60 * 1000; // 5 minutes
 function getCached(key) {
@@ -48,191 +81,175 @@ function setCache(key, result) {
     }
     cache.set(key, { result, expiresAt: Date.now() + CACHE_TTL });
 }
+async function runAgentQuery(params) {
+    const { prompt, schema, llmApiKey, llmProvider, llmModel, urls, sources: maxSources } = params;
+    const startMs = Date.now();
+    const numSources = Math.min(maxSources || 5, 10);
+    // Cache check
+    const cacheKey = `${prompt.trim()}:${JSON.stringify(schema || {})}`;
+    const cached = getCached(cacheKey);
+    if (cached)
+        return { ...cached, cached: true };
+    // Step 1: Resolve source URLs
+    let sourceUrls = [];
+    if (Array.isArray(urls) && urls.length > 0) {
+        sourceUrls = urls.map((u) => ({ url: u }));
+    }
+    else {
+        log.info(`Searching web for: "${prompt}"`);
+        const { provider, apiKey: searchApiKey } = getBestSearchProvider();
+        try {
+            const searchResults = await provider.searchWeb(prompt.trim(), { count: numSources, apiKey: searchApiKey });
+            sourceUrls = searchResults.slice(0, numSources).map((r) => ({ url: r.url, title: r.title, snippet: r.snippet }));
+        }
+        catch (err) {
+            log.warn('Search failed:', err.message);
+        }
+    }
+    if (sourceUrls.length === 0) {
+        return { success: false, error: { type: 'no_sources', message: 'Could not find relevant pages for this query' }, prompt, elapsed: Date.now() - startMs };
+    }
+    // Step 2: Fetch pages in parallel
+    log.info(`Fetching ${sourceUrls.length} sources in parallel`);
+    const PER_SOURCE_TIMEOUT_MS = 5000;
+    const fetchPromises = sourceUrls.map(async (source) => {
+        try {
+            const result = await Promise.race([
+                peel(source.url, { render: false, noEscalate: true, format: 'markdown', timeout: PER_SOURCE_TIMEOUT_MS, budget: 3000 }),
+                new Promise((_, reject) => setTimeout(() => reject(new Error('per-source timeout')), PER_SOURCE_TIMEOUT_MS)),
+            ]);
+            return { url: source.url, title: result.title || source.title || '', content: (result.content || '').slice(0, 15000), tokens: result.tokens || 0 };
+        }
+        catch {
+            return null;
+        }
+    });
+    const fetchResults = (await Promise.allSettled(fetchPromises))
+        .map((r) => (r.status === 'fulfilled' ? r.value : null))
+        .filter(Boolean);
+    if (fetchResults.length === 0) {
+        return { success: false, error: { type: 'fetch_failed', message: 'Could not fetch any of the found pages' }, prompt, sources: sourceUrls.map((s) => ({ url: s.url })), elapsed: Date.now() - startMs };
+    }
+    // Step 3: Extract or answer
+    const combinedContent = fetchResults.map((r) => `### ${r.title || r.url}\nURL: ${r.url}\n\n${r.content}`).join('\n\n---\n\n');
+    const totalTokens = fetchResults.reduce((sum, r) => sum + r.tokens, 0);
+    let result;
+    if (schema && llmApiKey) {
+        log.info('Using LLM extraction');
+        const extracted = await extractWithLLM({
+            content: combinedContent.slice(0, 30000), schema, llmApiKey, llmProvider: (llmProvider || 'openai'), llmModel,
+            prompt: `Based on these web pages, ${prompt}`, url: fetchResults[0].url,
+        });
+        const llmTokensUsed = (extracted.tokensUsed?.input ?? 0) + (extracted.tokensUsed?.output ?? 0);
+        result = { success: true, data: extracted.items, sources: fetchResults.map((r) => ({ url: r.url, title: r.title })), method: 'agent-llm',
+            llm: { provider: extracted.provider || llmProvider || 'openai', model: extracted.model || llmModel || 'default' }, tokensUsed: totalTokens + llmTokensUsed, elapsed: Date.now() - startMs };
+    }
+    else {
+        log.info('Using BM25 text extraction');
+        const qa = quickAnswer({ question: prompt, content: combinedContent, maxPassages: 3, maxChars: 2000 });
+        result = { success: true, answer: qa.answer || combinedContent.slice(0, 2000), confidence: qa.confidence ?? 0,
+            sources: fetchResults.map((r) => ({ url: r.url, title: r.title })), method: 'agent-bm25', tokensUsed: totalTokens, elapsed: Date.now() - startMs };
+    }
+    setCache(cacheKey, result);
+    return result;
+}
 // ---------------------------------------------------------------------------
 // Route factory
 // ---------------------------------------------------------------------------
 export function createAgentRouter() {
     const router = Router();
+    // ── POST /v1/agent — single query (with optional webhook) ──────────────
     router.post('/', async (req, res) => {
-        const { prompt, schema, llmApiKey, llmProvider, llmModel, urls, sources: maxSources, } = req.body || {};
-        // Validate required param
+        const { prompt, schema, llmApiKey, llmProvider, llmModel, urls, sources: maxSources, webhook } = req.body || {};
+        const requestId = req.requestId || crypto.randomUUID();
         if (!prompt?.trim()) {
             return res.status(400).json({
                 success: false,
-                error: {
-                    type: 'missing_prompt',
-                    message: 'Provide a prompt describing what you want to find',
-                    hint: 'POST /v1/agent { "prompt": "Find Stripe pricing plans" }',
-                    docs: 'https://webpeel.dev/docs/api-reference',
-                },
-                requestId: req.requestId || crypto.randomUUID(),
+                error: { type: 'missing_prompt', message: 'Provide a prompt describing what you want to find',
+                    hint: 'POST /v1/agent { "prompt": "Find Stripe pricing plans" }', docs: 'https://webpeel.dev/docs/api-reference' },
+                requestId,
             });
         }
-        const startMs = Date.now();
-        const numSources = Math.min(maxSources || 5, 10);
-        const requestId = req.requestId || crypto.randomUUID();
-        // Cache check
-        const cacheKey = `${prompt.trim()}:${JSON.stringify(schema || {})}`;
-        const cached = getCached(cacheKey);
-        if (cached) {
-            return res.json({ ...cached, cached: true, requestId });
+        // Async mode: webhook provided → return immediately, deliver result later
+        if (webhook) {
+            const jobId = crypto.randomUUID();
+            res.json({ success: true, id: jobId, status: 'processing', requestId });
+            // Fire-and-forget agent query + webhook delivery
+            runAgentQuery({ prompt, schema, llmApiKey, llmProvider, llmModel, urls, sources: maxSources })
+                .then((result) => sendWebhook(webhook, 'agent.completed', { id: jobId, ...result, requestId }))
+                .catch((err) => {
+                log.error('Async agent error:', err.message);
+                sendWebhook(webhook, 'agent.failed', { id: jobId, error: err.message, requestId }).catch(() => { });
+            });
+            return;
         }
+        // Synchronous mode: wait for result
         try {
-            // -----------------------------------------------------------------------
-            // Step 1: Resolve source URLs — use caller-provided or search the web
-            // -----------------------------------------------------------------------
-            let sourceUrls = [];
-            if (Array.isArray(urls) && urls.length > 0) {
-                sourceUrls = urls.map((u) => ({ url: u }));
-            }
-            else {
-                log.info(`Searching web for: "${prompt}"`);
-                const { provider, apiKey: searchApiKey } = getBestSearchProvider();
-                let searchResults = [];
-                try {
-                    searchResults = await provider.searchWeb(prompt.trim(), {
-                        count: numSources,
-                        apiKey: searchApiKey,
-                    });
-                }
-                catch (err) {
-                    log.warn('Search failed:', err.message);
-                }
-                sourceUrls = searchResults.slice(0, numSources).map((r) => ({
-                    url: r.url,
-                    title: r.title,
-                    snippet: r.snippet,
-                }));
-            }
-            if (sourceUrls.length === 0) {
-                return res.json({
-                    success: false,
-                    error: {
-                        type: 'no_sources',
-                        message: 'Could not find relevant pages for this query',
-                    },
-                    prompt,
-                    elapsed: Date.now() - startMs,
-                    requestId,
-                });
-            }
-            // -----------------------------------------------------------------------
-            // Step 2: Fetch pages in parallel (HTTP only, no browser, 5s timeout)
-            // -----------------------------------------------------------------------
-            log.info(`Fetching ${sourceUrls.length} sources in parallel`);
-            const PER_SOURCE_TIMEOUT_MS = 5000;
-            const fetchPromises = sourceUrls.map(async (source) => {
-                try {
-                    const result = await Promise.race([
-                        peel(source.url, {
-                            render: false,
-                            noEscalate: true,
-                            format: 'markdown',
-                            timeout: PER_SOURCE_TIMEOUT_MS,
-                            budget: 3000,
-                        }),
-                        new Promise((_, reject) => setTimeout(() => reject(new Error('per-source timeout')), PER_SOURCE_TIMEOUT_MS)),
-                    ]);
-                    return {
-                        url: source.url,
-                        title: result.title || source.title || '',
-                        content: (result.content || '').slice(0, 15000),
-                        tokens: result.tokens || 0,
-                    };
-                }
-                catch {
-                    return null;
-                }
-            });
-            const fetchSettled = await Promise.allSettled(fetchPromises);
-            const fetchResults = fetchSettled
-                .map((r) => (r.status === 'fulfilled' ? r.value : null))
-                .filter(Boolean);
-            if (fetchResults.length === 0) {
-                return res.json({
-                    success: false,
-                    error: {
-                        type: 'fetch_failed',
-                        message: 'Could not fetch any of the found pages',
-                    },
-                    prompt,
-                    sources: sourceUrls.map((s) => ({ url: s.url })),
-                    elapsed: Date.now() - startMs,
-                    requestId,
-                });
-            }
-            // -----------------------------------------------------------------------
-            // Step 3: Extract or answer
-            // -----------------------------------------------------------------------
-            const combinedContent = fetchResults
-                .map((r) => `### ${r.title || r.url}\nURL: ${r.url}\n\n${r.content}`)
-                .join('\n\n---\n\n');
-            const totalTokens = fetchResults.reduce((sum, r) => sum + r.tokens, 0);
-            let result;
-            if (schema && llmApiKey) {
-                // ── LLM extraction path ──────────────────────────────────────────────
-                log.info('Using LLM extraction');
-                const extracted = await extractWithLLM({
-                    content: combinedContent.slice(0, 30000),
-                    schema,
-                    llmApiKey,
-                    llmProvider: llmProvider || 'openai',
-                    llmModel,
-                    prompt: `Based on these web pages, ${prompt}`,
-                    url: fetchResults[0].url,
-                });
-                const llmTokensUsed = (extracted.tokensUsed?.input ?? 0) + (extracted.tokensUsed?.output ?? 0);
-                result = {
-                    success: true,
-                    data: extracted.items,
-                    sources: fetchResults.map((r) => ({ url: r.url, title: r.title })),
-                    method: 'agent-llm',
-                    llm: {
-                        provider: extracted.provider || llmProvider || 'openai',
-                        model: extracted.model || llmModel || 'default',
-                    },
-                    tokensUsed: totalTokens + llmTokensUsed,
-                    elapsed: Date.now() - startMs,
-                    requestId,
-                };
-            }
-            else {
-                // ── BM25 text answer path (no LLM needed) ───────────────────────────
-                log.info('Using BM25 text extraction');
-                const qa = quickAnswer({
-                    question: prompt,
-                    content: combinedContent,
-                    maxPassages: 3,
-                    maxChars: 2000,
-                });
-                result = {
-                    success: true,
-                    answer: qa.answer || combinedContent.slice(0, 2000),
-                    confidence: qa.confidence ?? 0,
-                    sources: fetchResults.map((r) => ({ url: r.url, title: r.title })),
-                    method: 'agent-bm25',
-                    tokensUsed: totalTokens,
-                    elapsed: Date.now() - startMs,
-                    requestId,
-                };
-            }
-            // Cache the result
-            setCache(cacheKey, result);
-            return res.json(result);
+            const result = await runAgentQuery({ prompt, schema, llmApiKey, llmProvider, llmModel, urls, sources: maxSources });
+            return res.json({ ...result, requestId });
         }
         catch (err) {
             log.error('Agent error:', err.message);
             return res.status(500).json({
-                success: false,
-                error: {
-                    type: 'agent_error',
-                    message: err.message || 'An unexpected error occurred',
-                },
-                prompt,
-                elapsed: Date.now() - startMs,
-                requestId,
+                success: false, error: { type: 'agent_error', message: err.message || 'An unexpected error occurred' },
+                prompt, elapsed: 0, requestId,
+            });
+        }
+    });
+    // ── POST /v1/agent/batch — parallel batch queries ─────────────────────
+    router.post('/batch', async (req, res) => {
+        const { prompts, schema, llmApiKey, llmProvider, llmModel, sources, webhook } = req.body || {};
+        const requestId = req.requestId || crypto.randomUUID();
+        if (!Array.isArray(prompts) || prompts.length === 0) {
+            return res.status(400).json({
+                success: false, error: { type: 'missing_prompts', message: 'Provide an array of prompts',
+                    hint: 'POST /v1/agent/batch { "prompts": ["Find X", "Find Y"] }' }, requestId,
+            });
+        }
+        if (prompts.length > 50) {
+            return res.status(400).json({
+                success: false, error: { type: 'too_many_prompts', message: `Max 50 prompts per batch (got ${prompts.length})` }, requestId,
             });
         }
+        const jobId = crypto.randomUUID();
+        const job = { id: jobId, status: 'processing', total: prompts.length, completed: 0, results: [], webhook, createdAt: Date.now() };
+        batchJobs.set(jobId, job);
+        // Return immediately, then process in background
+        res.json({ success: true, id: jobId, status: 'processing', total: prompts.length, requestId });
+        // Process in background with concurrency limit of 5
+        // eslint-disable-next-line @typescript-eslint/no-floating-promises
+        const sem = new Semaphore(5);
+        const tasks = prompts.map(async (prompt) => {
+            await sem.acquire();
+            try {
+                const result = await runAgentQuery({ prompt, schema, llmApiKey, llmProvider, llmModel, sources });
+                job.results.push({ prompt, success: !!result.success, answer: result.answer,
+                    data: result.data, sources: result.sources, method: result.method, elapsed: result.elapsed });
+            }
+            catch (err) {
+                job.results.push({ prompt, success: false, error: err.message });
+            }
+            finally {
+                job.completed++;
+                sem.release();
+            }
+        });
+        Promise.allSettled(tasks).then(() => {
+            job.status = job.results.every((r) => r.success) ? 'completed' : 'completed';
+            if (webhook) {
+                sendWebhook(webhook, 'agent.batch.completed', { id: jobId, total: job.total, completed: job.completed, results: job.results })
+                    .catch((err) => log.error('Batch webhook failed:', err.message));
+            }
+        });
+        return;
+    });
+    // ── GET /v1/agent/batch/:id — poll batch status ───────────────────────
+    router.get('/batch/:id', async (req, res) => {
+        const job = batchJobs.get(req.params.id);
+        if (!job) {
+            return res.status(404).json({ success: false, error: { type: 'not_found', message: 'Batch job not found or expired' } });
+        }
+        return res.json({ success: true, id: job.id, status: job.status, total: job.total, completed: job.completed, results: job.results });
     });
     return router;
 }

package/dist/server/routes/session.d.ts

@@ -6,9 +6,11 @@
  * POST   /v1/session/:id/navigate   → navigate to URL { url }
  * POST   /v1/session/:id/act        → execute PageActions array
  * GET    /v1/session/:id/screenshot → take screenshot (image/png)
+ * GET    /v1/session/:id/cookies    → export cookies from session context
+ * POST   /v1/session/:id/cookies    → inject cookies into session context
  * DELETE /v1/session/:id            → close session
  *
- * Use cases: login flows, multi-step automation, UI testing.
+ * Use cases: login flows, multi-step automation, UI testing, cookie persistence.
  * This is what Browserbase charges $500/mo for — built into WebPeel.
  */
 import { Router } from 'express';

package/dist/server/routes/session.js

@@ -6,9 +6,11 @@
  * POST   /v1/session/:id/navigate   → navigate to URL { url }
  * POST   /v1/session/:id/act        → execute PageActions array
  * GET    /v1/session/:id/screenshot → take screenshot (image/png)
+ * GET    /v1/session/:id/cookies    → export cookies from session context
+ * POST   /v1/session/:id/cookies    → inject cookies into session context
  * DELETE /v1/session/:id            → close session
  *
- * Use cases: login flows, multi-step automation, UI testing.
+ * Use cases: login flows, multi-step automation, UI testing, cookie persistence.
  * This is what Browserbase charges $500/mo for — built into WebPeel.
  */
 import { Router } from 'express';
@@ -17,13 +19,15 @@ import { normalizeActions, executeActions } from '../../core/actions.js';
 import { ANTI_DETECTION_ARGS, getRandomViewport, getRandomUserAgent, applyStealthScripts, } from '../../core/browser-pool.js';
 import { extractReadableContent } from '../../core/readability.js';
 const sessions = new Map();
-const SESSION_TTL_MS = 5 * 60 * 1000; // 5 minutes idle TTL
+const DEFAULT_SESSION_TTL_MS = 5 * 60 * 1000; // 5 minutes idle TTL (default)
+const MAX_SESSION_TTL_MS = 60 * 60 * 1000; // 60 minutes (persist / max)
+const MIN_SESSION_TTL_MS = 1 * 60 * 1000; // 1 minute minimum
 const MAX_SESSIONS_PER_USER = 3; // prevent abuse
 // Cleanup expired sessions every minute
 const _cleanupInterval = setInterval(() => {
     const now = Date.now();
     for (const [id, session] of sessions) {
-        if (now - session.lastUsedAt > SESSION_TTL_MS) {
+        if (now - session.lastUsedAt > session.ttlMs) {
             session.browser.close().catch(() => { });
             sessions.delete(id);
         }
@@ -73,7 +77,18 @@ function extractReadableText(html, url) {
 // ── Router ────────────────────────────────────────────────────────────────────
 export function createSessionRouter() {
     const router = Router();
-    // ── POST /v1/session — create session ────────────────────────────────────────
+    /**
+     * POST /v1/session — create a stateful browser session
+     *
+     * Body params:
+     *   url?     {string}  Initial URL to navigate to (optional).
+     *   ttl?     {number}  Session idle TTL in minutes (1–60, default 5).
+     *                      Timer resets on every request that touches the session.
+     *   persist? {boolean} Shorthand for ttl=60. Enables long-lived sessions
+     *                      for login flows where cookies must persist.
+     *
+     * Returns: { sessionId, currentUrl, expiresAt, ttlMinutes }
+     */
     router.post('/v1/session', async (req, res) => {
         const ownerId = getOwnerId(req);
         if (!ownerId) {
@@ -95,7 +110,15 @@ export function createSessionRouter() {
             });
             return;
         }
-        const { url } = req.body;
+        const { url, ttl, persist } = req.body;
+        // Resolve TTL: persist=true → 60 min max, ttl overrides default, clamp to [1, 60] min
+        let ttlMs = DEFAULT_SESSION_TTL_MS;
+        if (persist) {
+            ttlMs = MAX_SESSION_TTL_MS;
+        }
+        else if (typeof ttl === 'number') {
+            ttlMs = Math.min(MAX_SESSION_TTL_MS, Math.max(MIN_SESSION_TTL_MS, ttl * 60 * 1000));
+        }
         let browser = null;
         try {
             browser = await launchBrowser();
@@ -137,11 +160,13 @@ export function createSessionRouter() {
                 createdAt: now,
                 lastUsedAt: now,
                 currentUrl: page.url(),
+                ttlMs,
             });
             res.status(201).json({
                 sessionId: id,
                 currentUrl: page.url(),
-                expiresAt: new Date(now + SESSION_TTL_MS).toISOString(),
+                expiresAt: new Date(now + ttlMs).toISOString(),
+                ttlMinutes: ttlMs / 60_000,
             });
         }
         catch (err) {
@@ -188,7 +213,8 @@ export function createSessionRouter() {
                 currentUrl: session.page.url(),
                 title,
                 content,
-                expiresAt: new Date(session.lastUsedAt + SESSION_TTL_MS).toISOString(),
+                expiresAt: new Date(session.lastUsedAt + session.ttlMs).toISOString(),
+                ttlMinutes: session.ttlMs / 60_000,
             });
         }
         catch (err) {
@@ -242,7 +268,8 @@ export function createSessionRouter() {
             res.json({
                 currentUrl: session.page.url(),
                 title: await session.page.title(),
-                expiresAt: new Date(session.lastUsedAt + SESSION_TTL_MS).toISOString(),
+                expiresAt: new Date(session.lastUsedAt + session.ttlMs).toISOString(),
+                ttlMinutes: session.ttlMs / 60_000,
             });
         }
         catch (err) {
@@ -327,7 +354,8 @@ export function createSessionRouter() {
                 title,
                 screenshot,
                 actionsExecuted: normalizedActions.length,
-                expiresAt: new Date(session.lastUsedAt + SESSION_TTL_MS).toISOString(),
+                expiresAt: new Date(session.lastUsedAt + session.ttlMs).toISOString(),
+                ttlMinutes: session.ttlMs / 60_000,
             });
         }
         catch (err) {
@@ -367,6 +395,7 @@ export function createSessionRouter() {
             session.lastUsedAt = Date.now();
             res.setHeader('Content-Type', 'image/png');
             res.setHeader('Cache-Control', 'no-store');
+            res.setHeader('X-Session-Expires-At', new Date(session.lastUsedAt + session.ttlMs).toISOString());
             res.send(buf);
         }
         catch (err) {
@@ -382,6 +411,128 @@ export function createSessionRouter() {
             });
         }
     });
+    /**
+     * GET /v1/session/:id/cookies — export all cookies from the session's browser context
+     *
+     * Returns: { sessionId, cookies: Cookie[], count: number, expiresAt: string }
+     *
+     * Each cookie follows the Playwright Cookie shape:
+     *   { name, value, domain, path, expires, httpOnly, secure, sameSite }
+     *
+     * Use this to snapshot cookies after a login flow, then re-inject them later
+     * via POST /v1/session/:id/cookies to skip re-authentication.
+     */
+    router.get('/v1/session/:id/cookies', async (req, res) => {
+        const ownerId = getOwnerId(req);
+        const session = getSession(req.params['id'], ownerId);
+        if (!session) {
+            res.status(404).json({
+                success: false,
+                error: {
+                    type: 'session_not_found',
+                    message: 'Session not found or has expired.',
+                    hint: 'Create a new session via POST /v1/session.',
+                    docs: 'https://webpeel.dev/docs/errors#session-not-found',
+                },
+                requestId: req.requestId || randomUUID(),
+            });
+            return;
+        }
+        try {
+            // Playwright context.cookies() returns all cookies for all URLs by default
+            const cookies = await session.context.cookies();
+            session.lastUsedAt = Date.now();
+            res.json({
+                sessionId: session.id,
+                cookies,
+                count: cookies.length,
+                expiresAt: new Date(session.lastUsedAt + session.ttlMs).toISOString(),
+            });
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'cookie_export_failed',
+                    message: msg,
+                    docs: 'https://webpeel.dev/docs/errors#cookie-export-failed',
+                },
+                requestId: req.requestId || randomUUID(),
+            });
+        }
+    });
+    /**
+     * POST /v1/session/:id/cookies — inject cookies into the session's browser context
+     *
+     * Body params:
+     *   cookies {Cookie[]} Array of Playwright-compatible cookie objects.
+     *                      Required fields: name, value, domain (or url).
+     *                      Optional: path, expires, httpOnly, secure, sameSite.
+     *
+     * Returns: { sessionId, injected: number, expiresAt: string }
+     *
+     * Typical cookie-persistence workflow:
+     *   1. POST /v1/session { url: "https://example.com", persist: true }
+     *   2. POST /v1/session/:id/act  (complete login flow)
+     *   3. GET  /v1/session/:id/cookies  → save cookies array to your storage
+     *   4. Later: POST /v1/session/:id/cookies { cookies: [...] }
+     *   5. GET  /v1/session/:id  → page loads authenticated (no re-login needed)
+     */
+    router.post('/v1/session/:id/cookies', async (req, res) => {
+        const ownerId = getOwnerId(req);
+        const session = getSession(req.params['id'], ownerId);
+        if (!session) {
+            res.status(404).json({
+                success: false,
+                error: {
+                    type: 'session_not_found',
+                    message: 'Session not found or has expired.',
+                    hint: 'Create a new session via POST /v1/session.',
+                    docs: 'https://webpeel.dev/docs/errors#session-not-found',
+                },
+                requestId: req.requestId || randomUUID(),
+            });
+            return;
+        }
+        const { cookies } = req.body;
+        if (!Array.isArray(cookies) || cookies.length === 0) {
+            res.status(400).json({
+                success: false,
+                error: {
+                    type: 'bad_request',
+                    message: '`cookies` must be a non-empty array of cookie objects.',
+                    hint: 'Pass cookies exported from GET /v1/session/:id/cookies or a compatible Cookie[] array.',
+                    docs: 'https://webpeel.dev/docs/errors#bad-request',
+                },
+                requestId: req.requestId || randomUUID(),
+            });
+            return;
+        }
+        try {
+            // Playwright's addCookies validates the shape internally; invalid cookies will throw
+            await session.context.addCookies(cookies);
+            session.lastUsedAt = Date.now();
+            res.json({
+                sessionId: session.id,
+                injected: cookies.length,
+                expiresAt: new Date(session.lastUsedAt + session.ttlMs).toISOString(),
+            });
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            res.status(400).json({
+                success: false,
+                error: {
+                    type: 'cookie_inject_failed',
+                    message: msg,
+                    hint: 'Ensure each cookie has at minimum: name, value, and domain (or url).',
+                    docs: 'https://webpeel.dev/docs/errors#cookie-inject-failed',
+                },
+                requestId: req.requestId || randomUUID(),
+            });
+        }
+    });
     // ── DELETE /v1/session/:id ───────────────────────────────────────────────────
     router.delete('/v1/session/:id', async (req, res) => {
         const ownerId = getOwnerId(req);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.20.5",
+  "version": "0.20.7",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",