webpeel 0.20.6 → 0.20.8

package/dist/cli/commands/auth.js

@@ -3,6 +3,7 @@
  */
 import { handleLogin, handleLogout, handleUsage, loadConfig, saveConfig } from '../../cli-auth.js';
 import { clearCache, cacheStats } from '../../cache.js';
+import { loginToProfile } from '../../core/profiles.js';
 import { cliVersion } from '../utils.js';
 export function registerAuthCommands(program) {
     // ── auth command ──────────────────────────────────────────────────────────
@@ -254,13 +255,37 @@ export function registerAuthCommands(program) {
         console.log('   Try: webpeel "https://news.ycombinator.com" --json');
     });
     // ── login command ─────────────────────────────────────────────────────────
+    // Two modes:
+    //   webpeel login             — interactive API key authentication (existing)
+    //   webpeel login <domain>    — browser login: open site, log in, save cookies as profile
     program
-        .command('login')
-        .description('Authenticate the CLI with your API key')
-        .action(async () => {
+        .command('login [domain]')
+        .description('Authenticate: no args = API key auth; with domain = browser login (saves cookies as a named profile)')
+        .option('--profile <name>', 'Profile name to save under (defaults to the domain)')
+        .action(async (domain, opts) => {
         try {
-            await handleLogin();
-            process.exit(0);
+            if (domain) {
+                // ── Browser login mode ──────────────────────────────────────────
+                const url = domain.startsWith('http') ? domain : `https://${domain}`;
+                // Extract hostname for profile name default (e.g. "instagram.com" from "https://www.instagram.com/")
+                let defaultProfileName;
+                try {
+                    const hostname = new URL(url).hostname;
+                    // Strip "www." prefix for cleaner profile names
+                    defaultProfileName = hostname.replace(/^www\./, '');
+                }
+                catch {
+                    defaultProfileName = domain;
+                }
+                const profileName = opts.profile || defaultProfileName;
+                await loginToProfile(url, profileName);
+                process.exit(0);
+            }
+            else {
+                // ── API key auth mode (original behavior) ───────────────────────
+                await handleLogin();
+                process.exit(0);
+            }
         }
         catch (error) {
             console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);

package/dist/cli/commands/interact.js

@@ -310,6 +310,7 @@ export function registerInteractCommands(program) {
         .option('--schema <json>', 'Schema template name (e.g. product, article) or JSON schema for structured output')
         .option('-s, --silent', 'Silent mode (no spinner)')
         .option('--json', 'Output as JSON')
+        .option('--stream', 'Stream progress via SSE (calls API endpoint, requires API key)')
         .action(async (prompt, options) => {
         const llmApiKey = options.llmKey || process.env.OPENAI_API_KEY;
         const urls = options.urls ? options.urls.split(',').map((u) => u.trim()) : undefined;

package/dist/core/domain-extractors.js

@@ -324,19 +324,87 @@ async function twitterExtractor(html, url) {
                     source: 'fxtwitter',
                 };
                 // Try to fetch recent tweets from Twitter's public syndication endpoint
+                // NOTE: simpleFetch sends too many Sec-* headers that trigger 429. Use https directly.
                 let recentTweets = '';
                 try {
-                    const syndicationUrl = `https://syndication.twitter.com/srv/timeline-profile/screen-name/${u.screen_name}`;
-                    const syndicationResult = await simpleFetch(syndicationUrl, 'Mozilla/5.0 (compatible; WebPeel/1.0)', 8000);
-                    if (syndicationResult?.html) {
-                        // Extract tweet texts from the syndication HTML
-                        const tweetMatches = [...syndicationResult.html.matchAll(/"full_text":"((?:[^"\\]|\\.)*)"/g)];
-                        const tweets = tweetMatches
-                            .slice(0, 5)
-                            .map(m => m[1].replace(/\\n/g, ' ').replace(/\\"/g, '"').trim())
-                            .filter(t => t.length > 10 && !t.startsWith('RT @'));
-                        if (tweets.length > 0) {
-                            recentTweets = '\n\n### Recent Tweets\n\n' + tweets.map(t => `> ${t}`).join('\n\n');
+                    const { default: httpsModule } = await import('https');
+                    const syndicationHtml = await new Promise((resolve, reject) => {
+                        const req = httpsModule.request({
+                            hostname: 'syndication.twitter.com',
+                            path: `/srv/timeline-profile/screen-name/${u.screen_name}`,
+                            method: 'GET',
+                            headers: {
+                                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                                'Accept-Language': 'en-US,en;q=0.9',
+                            },
+                        }, (res) => {
+                            if (res.statusCode !== 200) {
+                                reject(new Error(`HTTP ${res.statusCode}`));
+                                res.resume();
+                                return;
+                            }
+                            let body = '';
+                            res.on('data', (chunk) => body += chunk.toString());
+                            res.on('end', () => resolve(body));
+                        });
+                        req.on('error', reject);
+                        setTimeout(() => req.destroy(new Error('timeout')), 12000);
+                        req.end();
+                    });
+                    if (syndicationHtml) {
+                        // Parse __NEXT_DATA__ JSON from the syndication page for rich tweet data
+                        const nextDataMatch = syndicationHtml.match(/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/i);
+                        if (nextDataMatch) {
+                            const nextData = tryParseJson(nextDataMatch[1]);
+                            const entries = nextData?.props?.pageProps?.timeline?.entries || [];
+                            const tweetSections = [];
+                            for (const entry of entries) {
+                                if (tweetSections.length >= 8)
+                                    break;
+                                const tweet = entry?.content?.tweet;
+                                if (!tweet?.full_text)
+                                    continue;
+                                const text = tweet.full_text.replace(/\\n/g, '\n').replace(/\\"/g, '"').trim();
+                                // Skip retweets and pure-URL-only tweets without media
+                                if (text.startsWith('RT @'))
+                                    continue;
+                                const media = tweet.extended_entities?.media || tweet.entities?.media || [];
+                                const isUrlOnly = /^https?:\/\/t\.co\/\S+$/.test(text.trim()) || /^https?:\/\/t\.co\/\S+\s*$/.test(text.trim());
+                                if (isUrlOnly && media.length === 0)
+                                    continue;
+                                // Format date
+                                const dateStr = tweet.created_at ? (() => {
+                                    try {
+                                        return new Date(tweet.created_at).toLocaleDateString('en-US', { year: 'numeric', month: 'long', day: 'numeric' });
+                                    }
+                                    catch {
+                                        return tweet.created_at;
+                                    }
+                                })() : '';
+                                const likes = tweet.favorite_count ?? 0;
+                                const retweets = tweet.retweet_count ?? 0;
+                                const replies = tweet.reply_count ?? 0;
+                                const fmtNum = (n) => n >= 1000000 ? (n / 1000000).toFixed(1) + 'M' : n >= 1000 ? (n / 1000).toFixed(1) + 'K' : String(n);
+                                const mediaLine = media.length > 0 ? `\n📷 ${media.map((m) => m.media_url_https || m.media_url).filter(Boolean).join(', ')}` : '';
+                                // Clean t.co URLs from text when they have real media
+                                const cleanText = media.length > 0 ? text.replace(/https?:\/\/t\.co\/\S+/g, '').trim() : text;
+                                tweetSections.push(`### ${dateStr}\n${cleanText}${mediaLine}\n♻️ ${fmtNum(retweets)} | ❤️ ${fmtNum(likes)} | 💬 ${fmtNum(replies)}`);
+                            }
+                            if (tweetSections.length > 0) {
+                                recentTweets = '\n\n## Recent Tweets\n\n' + tweetSections.join('\n\n---\n\n');
+                            }
+                        }
+                        else {
+                            // Fallback: simple regex extraction without metrics
+                            const tweetMatches = [...syndicationHtml.matchAll(/"full_text":"((?:[^"\\]|\\.)*)"/g)];
+                            const tweets = tweetMatches
+                                .slice(0, 5)
+                                .map(m => m[1].replace(/\\n/g, ' ').replace(/\\"/g, '"').trim())
+                                .filter(t => t.length > 10 && !t.startsWith('RT @'));
+                            if (tweets.length > 0) {
+                                recentTweets = '\n\n## Recent Tweets\n\n' + tweets.map(t => `> ${t}`).join('\n\n');
+                            }
                         }
                     }
                 }
@@ -344,7 +412,7 @@ async function twitterExtractor(html, url) {
                 const websiteLine = structured.website ? `\n🌐 ${structured.website}` : '';
                 const joinedLine = structured.created ? `\n📅 Joined: ${structured.created}` : '';
                 const likesLine = structured.likes ? `  |  ❤️ Likes: ${structured.likes?.toLocaleString() || 0}` : '';
-                const cleanContent = `## 🐦 @${(structured.handle || '').replace('@', '')} on X/Twitter\n\n**${structured.name}**${structured.verified ? ' ✓' : ''}\n\n${structured.bio || ''}\n\n📍 ${structured.location || 'N/A'}${websiteLine}${joinedLine}\n\n👥 **Followers:** ${structured.followers?.toLocaleString() || 0}  |  **Following:** ${structured.following?.toLocaleString() || 0}  |  **Tweets:** ${structured.tweets?.toLocaleString() || 0}${likesLine}${recentTweets}`;
+                const cleanContent = `# @${(structured.handle || '').replace('@', '')} on X/Twitter\n\n**${structured.name}**${structured.verified ? ' ✓' : ''}\n\n${structured.bio || ''}\n\n📍 ${structured.location || 'N/A'}${websiteLine}${joinedLine}\n👥 Followers: ${structured.followers?.toLocaleString() || 0} | Following: ${structured.following?.toLocaleString() || 0} | Tweets: ${structured.tweets?.toLocaleString() || 0}${likesLine}${recentTweets}`;
                 return { domain, type: 'profile', structured, cleanContent };
             }
         }
@@ -1961,7 +2029,16 @@ async function linkedinExtractor(html, url) {
     try {
         const { load } = await import('cheerio');
         const $ = load(html);
-        // LinkedIn SSR exposes some data in meta tags and JSON-LD
+        // Detect page type from URL first
+        const urlObj = new URL(url);
+        const pathParts = urlObj.pathname.split('/').filter(Boolean);
+        const pageType = pathParts[0] === 'company' ? 'company'
+            : pathParts[0] === 'in' ? 'profile'
+                : pathParts[0] === 'jobs' ? 'job'
+                    : 'page';
+        // Detect if we're on the authwall (LinkedIn redirects unauthenticated requests)
+        const isAuthwall = html.includes('authwall') || html.includes('Join LinkedIn') || html.includes('Sign in') && !html.includes('linkedin.com/in/');
+        // --- Try parsing meta tags / JSON-LD from the HTML ---
         let jsonLd = null;
         $('script[type="application/ld+json"]').each((_, el) => {
             if (jsonLd)
@@ -1974,30 +2051,83 @@ async function linkedinExtractor(html, url) {
         const ogTitle = $('meta[property="og:title"]').attr('content') || '';
         const ogDescription = $('meta[property="og:description"]').attr('content') || '';
         const ogImage = $('meta[property="og:image"]').attr('content') || '';
-        const name = jsonLd?.name || ogTitle.replace(/ \| LinkedIn$/, '').trim() || '';
+        const metaDescription = $('meta[name="description"]').attr('content') || '';
+        let name = jsonLd?.name || ogTitle.replace(/ \| LinkedIn$/, '').replace(/Sign Up \| LinkedIn$/, '').trim() || '';
+        // When on authwall, discard authwall-specific meta data
+        let headline = isAuthwall ? (jsonLd?.jobTitle || '') : (jsonLd?.jobTitle || metaDescription?.split('|')?.[0]?.trim() || ogDescription || '');
+        let description = isAuthwall ? (jsonLd?.description || '') : (jsonLd?.description || ogDescription || '');
+        let location = $('[class*="location"]').first().text().trim() || jsonLd?.address?.addressLocality || '';
+        // --- If authwall or no useful data, try direct HTTPS fetch with minimal headers ---
+        // LinkedIn returns rich og: meta tags when fetched with a plain browser UA (no Sec-Fetch-* noise)
+        if (!name || isAuthwall || name.toLowerCase().includes('sign up') || name.toLowerCase().includes('linkedin')) {
+            try {
+                const { default: httpsLI } = await import('https');
+                const { gunzip } = await import('zlib');
+                const linkedInHtml = await new Promise((resolve, reject) => {
+                    const req = httpsLI.request({
+                        hostname: 'www.linkedin.com',
+                        path: urlObj.pathname,
+                        method: 'GET',
+                        headers: {
+                            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                            'Accept': 'text/html,application/xhtml+xml',
+                            'Accept-Language': 'en-US,en;q=0.9',
+                            'Accept-Encoding': 'gzip, deflate',
+                        },
+                    }, (res) => {
+                        if (res.statusCode && res.statusCode >= 400) {
+                            reject(new Error(`HTTP ${res.statusCode}`));
+                            res.resume();
+                            return;
+                        }
+                        const chunks = [];
+                        res.on('data', (chunk) => chunks.push(chunk));
+                        res.on('end', () => {
+                            const buf = Buffer.concat(chunks);
+                            const enc = res.headers['content-encoding'] || '';
+                            if (enc === 'gzip') {
+                                gunzip(buf, (err, decoded) => err ? reject(err) : resolve(decoded.toString('utf8')));
+                            }
+                            else {
+                                resolve(buf.toString('utf8'));
+                            }
+                        });
+                    });
+                    req.on('error', reject);
+                    setTimeout(() => req.destroy(new Error('timeout')), 10000);
+                    req.end();
+                });
+                if (linkedInHtml) {
+                    const $li = load(linkedInHtml);
+                    const liOgTitle = $li('meta[property="og:title"]').attr('content') || '';
+                    const liOgDesc = $li('meta[property="og:description"]').attr('content') || '';
+                    // Only use if it has real profile data (not authwall)
+                    if (liOgTitle && !liOgTitle.toLowerCase().includes('sign up') && !liOgTitle.toLowerCase().includes('join linkedin')) {
+                        // "Name - Headline | LinkedIn" or "Name | LinkedIn"
+                        const titleParts = liOgTitle.replace(/ \| LinkedIn$/, '').split(/\s*[-–]\s*/);
+                        if (titleParts[0])
+                            name = titleParts[0].trim();
+                        if (titleParts[1])
+                            headline = titleParts[1].trim();
+                        if (liOgDesc)
+                            description = liOgDesc;
+                    }
+                }
+            }
+            catch { /* direct fetch optional */ }
+        }
         if (!name)
             return null;
-        const headline = jsonLd?.jobTitle ||
-            $('meta[name="description"]').attr('content')?.split('|')?.[0]?.trim() ||
-            ogDescription || '';
-        const description = jsonLd?.description || ogDescription || '';
-        // Try to detect page type from URL
-        const pathParts = new URL(url).pathname.split('/').filter(Boolean);
-        const pageType = pathParts[0] === 'company' ? 'company'
-            : pathParts[0] === 'in' ? 'profile'
-                : pathParts[0] === 'jobs' ? 'job'
-                    : 'page';
-        // Extract any visible structured info from the HTML
-        const location = $('[class*="location"]').first().text().trim() ||
-            jsonLd?.address?.addressLocality || '';
         const structured = {
             name, headline, description, location, pageType,
             image: ogImage, url,
         };
         const typeLine = pageType === 'company' ? '🏢' : pageType === 'profile' ? '👤' : '🔗';
         const locationLine = location ? `\n📍 ${location}` : '';
-        const headlineLine = headline ? `\n*${headline}*` : '';
-        const cleanContent = `# ${typeLine} ${name}${headlineLine}${locationLine}\n\n${description}`;
+        const headlineLine = headline && headline !== name ? `\n*${headline}*` : '';
+        const descriptionLine = description ? `\n\n${description}` : '';
+        const authNote = '\n\n⚠️ Full LinkedIn profiles require authentication. Use /v1/session to log in first.';
+        const cleanContent = `# ${typeLine} ${name} — LinkedIn${headlineLine}${locationLine}${descriptionLine}${authNote}`;
         return { domain: 'linkedin.com', type: pageType, structured, cleanContent };
     }
     catch {
@@ -2541,8 +2671,73 @@ async function soundcloudExtractor(_html, url) {
 // 29. Instagram extractor (oEmbed)
 // ---------------------------------------------------------------------------
 async function instagramExtractor(_html, url) {
+    const pathParts = new URL(url).pathname.split('/').filter(Boolean);
+    const contentType = pathParts[0] === 'p' ? 'post' : pathParts[0] === 'reel' ? 'reel' : pathParts[0] === 'tv' ? 'igtv' : pathParts.length === 1 ? 'profile' : 'post';
+    // --- Profile extraction via Instagram internal API (no auth needed) ---
+    if (contentType === 'profile' && pathParts.length === 1) {
+        const username = pathParts[0];
+        try {
+            const apiUrl = `https://www.instagram.com/api/v1/users/web_profile_info/?username=${encodeURIComponent(username)}`;
+            const igHeaders = {
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                'X-IG-App-ID': '936619743392459',
+                'Accept': '*/*',
+                'Referer': 'https://www.instagram.com/',
+            };
+            const apiResult = await simpleFetch(apiUrl, igHeaders['User-Agent'], 12000, igHeaders);
+            const data = tryParseJson(apiResult?.html || '');
+            const user = data?.data?.user;
+            if (user && user.username) {
+                const followers = user.edge_followed_by?.count ?? 0;
+                const following = user.edge_follow?.count ?? 0;
+                const postCount = user.edge_owner_to_timeline_media?.count ?? 0;
+                const fmtNum = (n) => n >= 1000000 ? (n / 1000000).toFixed(1) + 'M' : n >= 1000 ? (n / 1000).toFixed(1) + 'K' : String(n);
+                const structured = {
+                    username: user.username,
+                    fullName: user.full_name || '',
+                    bio: user.biography || '',
+                    followers,
+                    following,
+                    posts: postCount,
+                    verified: user.is_verified || false,
+                    isPrivate: user.is_private || false,
+                    profilePic: user.profile_pic_url_hd || user.profile_pic_url || '',
+                    externalUrl: user.external_url || (user.bio_links?.[0]?.url) || '',
+                    contentType: 'profile',
+                };
+                // Recent posts
+                const edges = user.edge_owner_to_timeline_media?.edges || [];
+                const postSections = [];
+                for (const edge of edges.slice(0, 6)) {
+                    const node = edge?.node;
+                    if (!node)
+                        continue;
+                    const caption = node.edge_media_to_caption?.edges?.[0]?.node?.text || '';
+                    const likes = node.edge_liked_by?.count ?? node.edge_media_preview_like?.count ?? 0;
+                    const comments = node.edge_media_to_comment?.count ?? 0;
+                    const isVideo = node.is_video;
+                    const mediaType = isVideo ? '🎬' : '📸';
+                    const timestamp = node.taken_at_timestamp ? new Date(node.taken_at_timestamp * 1000).toLocaleDateString('en-US', { year: 'numeric', month: 'long', day: 'numeric' }) : '';
+                    const imgUrl = node.thumbnail_src || node.display_url || '';
+                    const captionSnippet = caption ? caption.slice(0, 150) + (caption.length > 150 ? '…' : '') : '';
+                    postSections.push(`### ${mediaType} ${timestamp}\n${captionSnippet}\n❤️ ${fmtNum(likes)} | 💬 ${fmtNum(comments)}${imgUrl ? `\n🖼 ${imgUrl}` : ''}`);
+                }
+                const verifiedBadge = structured.verified ? ' ✓' : '';
+                const privateBadge = structured.isPrivate ? ' 🔒' : '';
+                const bioLine = structured.bio ? `\n\n${structured.bio}` : '';
+                const externalLine = structured.externalUrl ? `\n🌐 ${structured.externalUrl}` : '';
+                const postsSection = postSections.length > 0 ? '\n\n## Recent Posts\n\n' + postSections.join('\n\n---\n\n') : '';
+                const cleanContent = `# @${structured.username} on Instagram${verifiedBadge}${privateBadge}\n\n**${structured.fullName || structured.username}**${bioLine}${externalLine}\n\n👥 ${fmtNum(followers)} Followers | ${fmtNum(following)} Following | ${fmtNum(postCount)} Posts${postsSection}`;
+                return { domain: 'instagram.com', type: 'profile', structured, cleanContent };
+            }
+        }
+        catch (e) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'Instagram profile API failed:', e instanceof Error ? e.message : e);
+        }
+    }
+    // --- Post/Reel/IGTV: Try oEmbed API ---
     try {
-        // Instagram official oEmbed (no access token needed for basic data)
         const oembedUrl = `https://graph.facebook.com/v22.0/instagram_oembed?url=${encodeURIComponent(url)}&fields=title,author_name,provider_name,thumbnail_url`;
         const data = await fetchJson(oembedUrl);
         // Also try noembed.com as fallback
@@ -2553,8 +2748,6 @@ async function instagramExtractor(_html, url) {
         }
         if (!resolvedData || resolvedData.error)
             return null;
-        const pathParts = new URL(url).pathname.split('/').filter(Boolean);
-        const contentType = pathParts[0] === 'p' ? 'post' : pathParts[0] === 'reel' ? 'reel' : pathParts[0] === 'tv' ? 'igtv' : pathParts.length === 1 ? 'profile' : 'post';
         const structured = {
             title: resolvedData.title || '',
             author: resolvedData.author_name || '',
@@ -2563,7 +2756,7 @@ async function instagramExtractor(_html, url) {
             contentType,
             provider: 'Instagram',
         };
-        const typeEmoji = contentType === 'reel' ? '🎬' : contentType === 'post' ? '📸' : contentType === 'profile' ? '👤' : '📱';
+        const typeEmoji = contentType === 'reel' ? '🎬' : contentType === 'post' ? '📸' : '📱';
         const titleText = structured.title || `Instagram ${contentType} by ${structured.author}`;
         const cleanContent = `## ${typeEmoji} Instagram ${contentType}: ${titleText}\n\n**Creator:** @${structured.author.replace('@', '')}\n**URL:** ${url}`;
         return { domain: 'instagram.com', type: contentType, structured, cleanContent };
@@ -2575,59 +2768,106 @@ async function instagramExtractor(_html, url) {
     }
 }
 // ---------------------------------------------------------------------------
-// 30. PDF extractor (URL-based detection)
+// 30. PDF extractor (URL-based detection) — downloads and extracts real text
 // ---------------------------------------------------------------------------
+const PDF_MAX_BYTES = 50 * 1024 * 1024; // 50 MB
+const PDF_TRUNCATE_CHARS = 100_000;
 async function pdfExtractor(_html, url) {
     try {
         const urlObj = new URL(url);
         const filename = urlObj.pathname.split('/').pop() || 'document.pdf';
         const hostname = urlObj.hostname;
-        // Try to get HEAD request metadata
-        let contentType = 'application/pdf';
-        let contentLength = '';
+        // Download the PDF
+        let buffer;
+        let finalContentType = 'application/pdf';
         try {
-            const { default: https } = await import('https');
-            const { default: http } = await import('http');
-            const client = url.startsWith('https') ? https : http;
-            await new Promise((resolve) => {
-                const req = client.request(url, { method: 'HEAD', timeout: 5000 }, (res) => {
-                    contentType = res.headers['content-type'] || 'application/pdf';
-                    contentLength = res.headers['content-length'] || '';
-                    resolve();
-                    res.resume();
-                });
-                req.on('error', () => resolve());
-                req.on('timeout', () => { req.destroy(); resolve(); });
-                req.end();
+            const response = await fetch(url, {
+                headers: { 'User-Agent': 'Mozilla/5.0 (compatible; WebPeel/1.0)' },
+                signal: AbortSignal.timeout(30000),
             });
+            if (!response.ok) {
+                if (process.env.DEBUG)
+                    console.debug('[webpeel]', `PDF download failed: HTTP ${response.status}`);
+                return null; // Let the normal pipeline handle it
+            }
+            finalContentType = response.headers.get('content-type') || 'application/pdf';
+            // Verify it's actually a PDF (content-type or URL)
+            const isPdf = finalContentType.toLowerCase().includes('pdf') || /\.pdf(\?|$|#)/i.test(url);
+            if (!isPdf)
+                return null;
+            const arrayBuffer = await response.arrayBuffer();
+            buffer = Buffer.from(arrayBuffer);
+        }
+        catch (downloadErr) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'PDF download error:', downloadErr instanceof Error ? downloadErr.message : downloadErr);
+            return null; // Let the normal pipeline handle it
         }
-        catch { /* best-effort */ }
-        const isPdf = contentType.toLowerCase().includes('pdf') || /\.pdf(\?|$|#)/i.test(url);
-        if (!isPdf)
+        // Size guard
+        if (buffer.length > PDF_MAX_BYTES) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', `PDF too large (${buffer.length} bytes), falling back to stub`);
             return null;
-        const fileSizeKb = contentLength ? Math.round(parseInt(contentLength) / 1024) : null;
-        const structured = {
-            filename,
-            url,
-            contentType,
-            fileSizeKb,
-            hostname,
-            note: 'PDF binary content — text extraction requires a PDF parser. Use a dedicated PDF extraction service for full text content.',
+        }
+        // Extract text via pdf-parse
+        const { extractPdf } = await import('./pdf.js');
+        let pdf;
+        try {
+            pdf = await extractPdf(buffer);
+        }
+        catch (parseErr) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'PDF parse failed:', parseErr instanceof Error ? parseErr.message : parseErr);
+            return null; // Let the normal pipeline handle it
+        }
+        // Normalize whitespace (pdf-parse emits lots of blank lines)
+        let text = (pdf.text || '')
+            .replace(/\r\n/g, '\n')
+            .replace(/\n{3,}/g, '\n\n')
+            .replace(/[ \t]+/g, ' ')
+            .trim();
+        // Truncate very large documents
+        let truncated = false;
+        if (text.length > PDF_TRUNCATE_CHARS) {
+            text = text.slice(0, PDF_TRUNCATE_CHARS);
+            truncated = true;
+        }
+        if (!text) {
+            // Scanned/image-only PDF — return a clear message rather than empty content
+            const emptyNote = `## 📄 ${filename}\n\n*This PDF appears to be a scanned document (image-only). No extractable text was found.*\n\n**Source:** ${url}`;
+            return {
+                domain: hostname,
+                type: 'pdf',
+                structured: { title: filename, url, pages: pdf.pages, contentType: finalContentType },
+                cleanContent: emptyNote,
+            };
+        }
+        // Build markdown output
+        const titleRaw = pdf.metadata?.title || '';
+        const title = titleRaw || filename.replace(/\.pdf$/i, '') || 'PDF Document';
+        const metaParts = [];
+        if (pdf.metadata?.author)
+            metaParts.push(`**Author:** ${pdf.metadata.author}`);
+        if (pdf.pages)
+            metaParts.push(`**Pages:** ${pdf.pages}`);
+        metaParts.push(`**Source:** ${url}`);
+        const header = titleRaw ? `# ${titleRaw}\n\n` : '';
+        const metaBlock = metaParts.join(' | ') + '\n\n';
+        const truncNote = truncated ? '\n\n*[Content truncated — document exceeds 100,000 characters]*' : '';
+        const cleanContent = header + metaBlock + text + truncNote;
+        return {
+            domain: hostname,
+            type: 'pdf',
+            structured: {
+                title,
+                filename,
+                url,
+                pages: pdf.pages,
+                contentType: finalContentType,
+                ...pdf.metadata,
+            },
+            cleanContent,
         };
-        const sizeStr = fileSizeKb ? ` (${fileSizeKb > 1024 ? (fileSizeKb / 1024).toFixed(1) + ' MB' : fileSizeKb + ' KB'})` : '';
-        const cleanContent = `## 📄 PDF Document: ${filename}
-
-**URL:** ${url}
-**Host:** ${hostname}${sizeStr ? `\n**Size:** ${sizeStr}` : ''}
-
-> **Note:** This is a PDF document. Binary PDF content cannot be directly extracted as text through standard web fetching. To extract the full text, consider:
->
-> 1. Use a dedicated PDF extraction service (e.g., Adobe PDF Extract API, pdfminer, PyMuPDF)
-> 2. Download the file and process locally with \`pdf-parse\` (Node.js) or \`pdfplumber\` (Python)
-> 3. For academic PDFs, check if an HTML version is available at the same URL without \`.pdf\`
-
-**Direct download URL:** ${url}`;
-        return { domain: hostname, type: 'pdf', structured, cleanContent };
     }
     catch (e) {
         if (process.env.DEBUG)

package/dist/core/profiles.d.ts

@@ -14,7 +14,8 @@ export interface ProfileMetadata {
     description?: string;
 }
 /**
- * Valid profile names: letters, digits, hyphens only. No spaces or special chars.
+ * Valid profile names: letters, digits, hyphens, and dots. No spaces or special chars.
+ * Dots are allowed so domain names like "instagram.com" work as profile names.
  */
 export declare function isValidProfileName(name: string): boolean;
 /**
@@ -45,3 +46,16 @@ export declare function deleteProfile(name: string): boolean;
  * 3. On browser close or Ctrl+C, captures storage state and saves the profile
  */
 export declare function createProfile(name: string, description?: string): Promise<void>;
+/**
+ * Open a headed browser, navigate to `url`, and wait for the user to log in.
+ * Pressing Enter (or closing the browser) saves the session as a named profile.
+ *
+ * Unlike `createProfile()` (which opens to about:blank and waits for browser close),
+ * this function:
+ *   1. Navigates directly to the given URL on launch
+ *   2. Waits for the user to press Enter (or close the browser) to save
+ *   3. Saves storage state AND creates metadata under ~/.webpeel/profiles/<name>/
+ *
+ * Profile names may contain letters, digits, hyphens, and dots (e.g. "instagram.com").
+ */
+export declare function loginToProfile(url: string, profileName: string, description?: string): Promise<void>;

package/dist/core/profiles.js

@@ -19,10 +19,11 @@ function ensureProfilesDir() {
 }
 // ─── Name validation ─────────────────────────────────────────────────────────
 /**
- * Valid profile names: letters, digits, hyphens only. No spaces or special chars.
+ * Valid profile names: letters, digits, hyphens, and dots. No spaces or special chars.
+ * Dots are allowed so domain names like "instagram.com" work as profile names.
  */
 export function isValidProfileName(name) {
-    return /^[a-zA-Z0-9-]+$/.test(name) && name.length > 0 && name.length <= 64;
+    return /^[a-zA-Z0-9\-.]+$/.test(name) && name.length > 0 && name.length <= 64;
 }
 // ─── Core helpers ─────────────────────────────────────────────────────────────
 /**
@@ -213,3 +214,137 @@ export async function createProfile(name, description) {
         });
     });
 }
+// ─── Browser-based login helper ───────────────────────────────────────────────
+/**
+ * Open a headed browser, navigate to `url`, and wait for the user to log in.
+ * Pressing Enter (or closing the browser) saves the session as a named profile.
+ *
+ * Unlike `createProfile()` (which opens to about:blank and waits for browser close),
+ * this function:
+ *   1. Navigates directly to the given URL on launch
+ *   2. Waits for the user to press Enter (or close the browser) to save
+ *   3. Saves storage state AND creates metadata under ~/.webpeel/profiles/<name>/
+ *
+ * Profile names may contain letters, digits, hyphens, and dots (e.g. "instagram.com").
+ */
+export async function loginToProfile(url, profileName, description) {
+    if (!isValidProfileName(profileName)) {
+        throw new Error(`Invalid profile name "${profileName}". Use only letters, numbers, hyphens, and dots (no spaces).`);
+    }
+    ensureProfilesDir();
+    const profileDir = path.join(PROFILES_DIR, profileName);
+    const isUpdate = existsSync(profileDir) && existsSync(path.join(profileDir, 'metadata.json'));
+    mkdirSync(profileDir, { recursive: true });
+    const browser = await chromium.launch({ headless: false });
+    const context = await browser.newContext();
+    const page = await context.newPage();
+    try {
+        await page.goto(url);
+    }
+    catch (e) {
+        // Non-fatal — browser is open, user can navigate manually
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'initial navigation error:', e instanceof Error ? e.message : e);
+    }
+    console.log('');
+    console.log('╔══════════════════════════════════════════════════════╗');
+    console.log(`║  WebPeel Browser Login`);
+    console.log(`║  URL:     ${url}`);
+    console.log(`║  Profile: ${profileName}`);
+    console.log('║                                                      ║');
+    console.log('║  Log in, then press Enter here to save your session. ║');
+    console.log('║  (Or close the browser window — same effect.)        ║');
+    console.log('╚══════════════════════════════════════════════════════╝');
+    console.log('');
+    let saved = false;
+    const saveAndClose = async () => {
+        if (saved)
+            return;
+        saved = true;
+        console.log('\nCapturing browser session...');
+        try {
+            const storageState = await context.storageState();
+            writeFileSync(path.join(profileDir, 'storage-state.json'), JSON.stringify(storageState, null, 2));
+            // Extract unique domains from cookies (strip leading dot)
+            const domains = [
+                ...new Set((storageState.cookies ?? [])
+                    .map((c) => (c.domain ?? '').replace(/^\./, ''))
+                    .filter(Boolean)),
+            ];
+            const now = new Date().toISOString();
+            const meta = isUpdate
+                ? {
+                    // Preserve original creation date on update
+                    ...((() => {
+                        try {
+                            return JSON.parse(readFileSync(path.join(profileDir, 'metadata.json'), 'utf-8'));
+                        }
+                        catch {
+                            return {};
+                        }
+                    })()),
+                    name: profileName,
+                    lastUsed: now,
+                    domains,
+                    ...(description ? { description } : {}),
+                }
+                : {
+                    name: profileName,
+                    created: now,
+                    lastUsed: now,
+                    domains,
+                    ...(description ? { description } : {}),
+                };
+            writeFileSync(path.join(profileDir, 'metadata.json'), JSON.stringify(meta, null, 2));
+            console.log(`✅ Profile "${profileName}" ${isUpdate ? 'updated' : 'saved'}!`);
+            if (domains.length > 0) {
+                console.log(`   Domains: ${domains.join(', ')}`);
+            }
+            else {
+                console.log('   No login sessions detected (no cookies captured).');
+                console.log('   Make sure you completed the login before pressing Enter.');
+            }
+            console.log('');
+            console.log(`   Use with: webpeel "${url}" --profile ${profileName}`);
+        }
+        catch (e) {
+            console.error('Warning: Failed to save storage state:', e instanceof Error ? e.message : String(e));
+            // Clean up partial directory if this was a new profile
+            if (!isUpdate) {
+                try {
+                    rmSync(profileDir, { recursive: true, force: true });
+                }
+                catch {
+                    // ignore cleanup errors
+                }
+            }
+        }
+        try {
+            await browser.close();
+        }
+        catch {
+            // ignore close errors
+        }
+    };
+    // Three ways to save: Enter key, browser close, or Ctrl+C
+    await new Promise((resolve) => {
+        let resolved = false;
+        const done = async () => {
+            if (resolved)
+                return;
+            resolved = true;
+            await saveAndClose();
+            resolve();
+        };
+        // Wait for Enter key on stdin
+        if (process.stdin.isTTY) {
+            process.stdin.setRawMode(false);
+        }
+        process.stdin.resume();
+        process.stdin.once('data', () => done());
+        // Browser closed by user
+        browser.on('disconnected', () => done());
+        // Ctrl+C
+        process.once('SIGINT', () => done());
+    });
+}

package/dist/server/routes/agent.d.ts

@@ -14,6 +14,7 @@
  * Returns: { success, data|answer, sources, method, elapsed, tokensUsed }
  *
  * Webhook support: pass `webhook` URL to get async delivery with HMAC-SHA256 signing.
+ * Streaming support: pass `stream: true` to get SSE events instead of polling.
  *
  * 5-minute in-memory cache. Max 10 sources per request.
  */

package/dist/server/routes/agent.js

@@ -14,6 +14,7 @@
  * Returns: { success, data|answer, sources, method, elapsed, tokensUsed }
  *
  * Webhook support: pass `webhook` URL to get async delivery with HMAC-SHA256 signing.
+ * Streaming support: pass `stream: true` to get SSE events instead of polling.
  *
  * 5-minute in-memory cache. Max 10 sources per request.
  */
@@ -81,8 +82,14 @@ function setCache(key, result) {
     }
     cache.set(key, { result, expiresAt: Date.now() + CACHE_TTL });
 }
+// ---------------------------------------------------------------------------
+// SSE helpers
+// ---------------------------------------------------------------------------
+function sseWrite(res, event, data) {
+    res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
+}
 async function runAgentQuery(params) {
-    const { prompt, schema, llmApiKey, llmProvider, llmModel, urls, sources: maxSources } = params;
+    const { prompt, schema, llmApiKey, llmProvider, llmModel, urls, sources: maxSources, onSearching, onFetching, onExtracting } = params;
     const startMs = Date.now();
     const numSources = Math.min(maxSources || 5, 10);
     // Cache check
@@ -97,6 +104,8 @@ async function runAgentQuery(params) {
     }
     else {
         log.info(`Searching web for: "${prompt}"`);
+        if (onSearching)
+            onSearching();
         const { provider, apiKey: searchApiKey } = getBestSearchProvider();
         try {
             const searchResults = await provider.searchWeb(prompt.trim(), { count: numSources, apiKey: searchApiKey });
@@ -111,6 +120,8 @@ async function runAgentQuery(params) {
     }
     // Step 2: Fetch pages in parallel
     log.info(`Fetching ${sourceUrls.length} sources in parallel`);
+    if (onFetching)
+        onFetching(sourceUrls.length);
     const PER_SOURCE_TIMEOUT_MS = 5000;
     const fetchPromises = sourceUrls.map(async (source) => {
         try {
@@ -136,6 +147,8 @@ async function runAgentQuery(params) {
     let result;
     if (schema && llmApiKey) {
         log.info('Using LLM extraction');
+        if (onExtracting)
+            onExtracting('llm');
         const extracted = await extractWithLLM({
             content: combinedContent.slice(0, 30000), schema, llmApiKey, llmProvider: (llmProvider || 'openai'), llmModel,
             prompt: `Based on these web pages, ${prompt}`, url: fetchResults[0].url,
@@ -146,6 +159,8 @@ async function runAgentQuery(params) {
     }
     else {
         log.info('Using BM25 text extraction');
+        if (onExtracting)
+            onExtracting('bm25');
         const qa = quickAnswer({ question: prompt, content: combinedContent, maxPassages: 3, maxChars: 2000 });
         result = { success: true, answer: qa.answer || combinedContent.slice(0, 2000), confidence: qa.confidence ?? 0,
             sources: fetchResults.map((r) => ({ url: r.url, title: r.title })), method: 'agent-bm25', tokensUsed: totalTokens, elapsed: Date.now() - startMs };
@@ -158,9 +173,9 @@ async function runAgentQuery(params) {
 // ---------------------------------------------------------------------------
 export function createAgentRouter() {
     const router = Router();
-    // ── POST /v1/agent — single query (with optional webhook) ──────────────
+    // ── POST /v1/agent — single query (with optional webhook or stream) ──────
     router.post('/', async (req, res) => {
-        const { prompt, schema, llmApiKey, llmProvider, llmModel, urls, sources: maxSources, webhook } = req.body || {};
+        const { prompt, schema, llmApiKey, llmProvider, llmModel, urls, sources: maxSources, webhook, stream } = req.body || {};
         const requestId = req.requestId || crypto.randomUUID();
         if (!prompt?.trim()) {
             return res.status(400).json({
@@ -170,6 +185,34 @@ export function createAgentRouter() {
                 requestId,
             });
         }
+        // ── Streaming mode (SSE) ─────────────────────────────────────────────
+        if (stream === true) {
+            res.setHeader('Content-Type', 'text/event-stream');
+            res.setHeader('Cache-Control', 'no-cache');
+            res.setHeader('Connection', 'keep-alive');
+            res.setHeader('X-Accel-Buffering', 'no');
+            res.flushHeaders();
+            try {
+                const result = await runAgentQuery({
+                    prompt, schema, llmApiKey, llmProvider, llmModel, urls, sources: maxSources,
+                    onSearching: () => {
+                        sseWrite(res, 'searching', { message: 'Searching the web...' });
+                    },
+                    onFetching: (count) => {
+                        sseWrite(res, 'fetching', { message: `Fetching ${count} sources...`, count });
+                    },
+                    onExtracting: (method) => {
+                        sseWrite(res, 'extracting', { message: method === 'llm' ? 'Extracting with LLM...' : 'Analyzing with BM25...', method });
+                    },
+                });
+                sseWrite(res, 'done', { ...result, requestId });
+            }
+            catch (err) {
+                sseWrite(res, 'error', { message: err.message || 'An unexpected error occurred', requestId });
+            }
+            res.end();
+            return;
+        }
         // Async mode: webhook provided → return immediately, deliver result later
         if (webhook) {
             const jobId = crypto.randomUUID();
@@ -198,7 +241,7 @@ export function createAgentRouter() {
     });
     // ── POST /v1/agent/batch — parallel batch queries ─────────────────────
     router.post('/batch', async (req, res) => {
-        const { prompts, schema, llmApiKey, llmProvider, llmModel, sources, webhook } = req.body || {};
+        const { prompts, schema, llmApiKey, llmProvider, llmModel, sources, webhook, stream } = req.body || {};
         const requestId = req.requestId || crypto.randomUUID();
         if (!Array.isArray(prompts) || prompts.length === 0) {
             return res.status(400).json({
@@ -214,7 +257,57 @@ export function createAgentRouter() {
         const jobId = crypto.randomUUID();
         const job = { id: jobId, status: 'processing', total: prompts.length, completed: 0, results: [], webhook, createdAt: Date.now() };
         batchJobs.set(jobId, job);
-        // Return immediately, then process in background
+        // ── Streaming mode (SSE) — keep connection open ──────────────────────
+        if (stream === true) {
+            res.setHeader('Content-Type', 'text/event-stream');
+            res.setHeader('Cache-Control', 'no-cache');
+            res.setHeader('Connection', 'keep-alive');
+            res.setHeader('X-Accel-Buffering', 'no');
+            res.flushHeaders();
+            // Send start event
+            sseWrite(res, 'start', { id: jobId, total: prompts.length, requestId });
+            const sem = new Semaphore(5);
+            const tasks = prompts.map(async (prompt) => {
+                await sem.acquire();
+                try {
+                    const result = await runAgentQuery({ prompt, schema, llmApiKey, llmProvider, llmModel, sources });
+                    const entry = {
+                        prompt,
+                        success: !!result.success,
+                        answer: result.answer,
+                        data: result.data,
+                        sources: result.sources,
+                        method: result.method,
+                        elapsed: result.elapsed,
+                    };
+                    job.results.push(entry);
+                    job.completed++;
+                    // Send per-prompt progress event
+                    sseWrite(res, 'progress', { completed: job.completed, total: job.total, result: entry });
+                }
+                catch (err) {
+                    const entry = { prompt, success: false, error: err.message };
+                    job.results.push(entry);
+                    job.completed++;
+                    sseWrite(res, 'progress', { completed: job.completed, total: job.total, result: entry });
+                }
+                finally {
+                    sem.release();
+                }
+            });
+            await Promise.allSettled(tasks);
+            job.status = 'completed';
+            // Send done event
+            sseWrite(res, 'done', { id: jobId, total: job.total, completed: job.completed, requestId });
+            res.end();
+            // Fire webhook if configured
+            if (webhook) {
+                sendWebhook(webhook, 'agent.batch.completed', { id: jobId, total: job.total, completed: job.completed, results: job.results })
+                    .catch((err) => log.error('Batch webhook failed:', err.message));
+            }
+            return;
+        }
+        // Non-streaming mode: Return immediately, then process in background
         res.json({ success: true, id: jobId, status: 'processing', total: prompts.length, requestId });
         // Process in background with concurrency limit of 5
         // eslint-disable-next-line @typescript-eslint/no-floating-promises

package/dist/server/routes/crawl.js

@@ -11,12 +11,14 @@ import { Router } from 'express';
 import '../types.js'; // Augments Express.Request with requestId
 import { crawl } from '../../core/crawler.js';
 import { validateUrlForSSRF, SSRFError } from '../middleware/url-validator.js';
+import crypto from 'crypto';
 export function createCrawlRouter(jobQueue) {
     const router = Router();
     /**
      * POST /v1/crawl
      *
      * Start an async crawl job. Returns a job ID immediately; poll GET /v1/crawl/:id for status.
+     * With stream:true, keeps the connection open and sends SSE events per page.
      *
      * Body:
      *   url            {string}   Required. Starting URL.
@@ -26,10 +28,11 @@ export function createCrawlRouter(jobQueue) {
      *   excludePatterns {string[]} Regex patterns — skip matching URLs.
      *   formats        {string[]} Content formats: 'markdown' | 'text' (default: ['markdown']).
      *   webhook        {object}   Optional webhook to POST results to when done.
+     *   stream         {boolean}  If true, respond with SSE events (start → progress → done).
      */
     router.post('/', async (req, res) => {
         try {
-            const { url, maxPages = 10, maxDepth = 2, includePatterns = [], excludePatterns = [], webhook, } = req.body ?? {};
+            const { url, maxPages = 10, maxDepth = 2, includePatterns = [], excludePatterns = [], webhook, stream, } = req.body ?? {};
             // Validate URL
             if (!url || typeof url !== 'string') {
                 res.status(400).json({
@@ -78,6 +81,76 @@ export function createCrawlRouter(jobQueue) {
                 throw error;
             }
             const ownerId = req.auth?.keyInfo?.accountId;
+            // ── Streaming mode (SSE) — keep connection open ──────────────────────
+            if (stream === true) {
+                res.setHeader('Content-Type', 'text/event-stream');
+                res.setHeader('Cache-Control', 'no-cache');
+                res.setHeader('Connection', 'keep-alive');
+                res.setHeader('X-Accel-Buffering', 'no');
+                res.flushHeaders();
+                const jobId = crypto.randomUUID();
+                // Send start event (total unknown until crawl runs)
+                res.write(`event: start\ndata: ${JSON.stringify({ id: jobId, url, maxPages, requestId: req.requestId })}\n\n`);
+                const crawlOptions = {
+                    maxPages,
+                    maxDepth,
+                    tier: req.auth?.tier,
+                    onProgress: (progress) => {
+                        const total = progress.crawled + progress.queued;
+                        res.write(`event: progress\ndata: ${JSON.stringify({
+                            id: jobId,
+                            completed: progress.crawled,
+                            total,
+                            queued: progress.queued,
+                            currentUrl: progress.currentUrl,
+                        })}\n\n`);
+                    },
+                };
+                if (Array.isArray(includePatterns) && includePatterns.length > 0) {
+                    crawlOptions.includePatterns = includePatterns;
+                }
+                if (Array.isArray(excludePatterns) && excludePatterns.length > 0) {
+                    crawlOptions.excludePatterns = excludePatterns;
+                }
+                try {
+                    const results = await crawl(url, crawlOptions);
+                    const data = results.map(r => ({
+                        url: r.url,
+                        title: r.title,
+                        content: r.markdown,
+                        links: r.links,
+                        elapsed: r.elapsed,
+                    }));
+                    res.write(`event: done\ndata: ${JSON.stringify({
+                        id: jobId,
+                        total: results.length,
+                        completed: results.length,
+                        results: data,
+                        requestId: req.requestId,
+                    })}\n\n`);
+                    // Fire webhook if configured
+                    if (webhook) {
+                        Promise.resolve(jobQueue.createJob('crawl', webhook, ownerId)).then((job) => {
+                            jobQueue.updateJob(job.id, {
+                                status: 'completed',
+                                data,
+                                total: results.length,
+                                completed: results.length,
+                                creditsUsed: results.length,
+                            });
+                        }).catch(() => { });
+                    }
+                }
+                catch (error) {
+                    res.write(`event: error\ndata: ${JSON.stringify({
+                        id: jobId,
+                        message: error.message || 'Crawl failed',
+                        requestId: req.requestId,
+                    })}\n\n`);
+                }
+                res.end();
+                return;
+            }
             const job = await jobQueue.createJob('crawl', webhook, ownerId);
             // Start crawl in background
             setImmediate(async () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.20.6",
+  "version": "0.20.8",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",