webpeel 0.8.1 → 0.9.0

package/dist/cli.js

@@ -18,6 +18,8 @@ import { writeFileSync, readFileSync } from 'fs';
 import { peel, peelBatch, cleanup } from './index.js';
 import { checkUsage, showUsageFooter, handleLogin, handleLogout, handleUsage, loadConfig, saveConfig } from './cli-auth.js';
 import { getCache, setCache, parseTTL, clearCache, cacheStats } from './cache.js';
+import { estimateTokens } from './core/markdown.js';
+import { distillToBudget, budgetListings } from './core/budget.js';
 const program = new Command();
 // Read version from package.json dynamically
 import { fileURLToPath } from 'url';
@@ -115,6 +117,8 @@ function parseActions(actionStrings) {
                 return { type: 'hover', selector: value };
             case 'waitFor':
                 return { type: 'waitForSelector', selector: value };
+            case 'wait-for':
+                return { type: 'waitForSelector', selector: value, timeout: 10000 };
             case 'screenshot':
                 return { type: 'screenshot' };
             default:
@@ -130,7 +134,7 @@ program
     .option('--html', 'Output raw HTML instead of markdown')
     .option('--text', 'Output plain text instead of markdown')
     .option('--json', 'Output as JSON')
-    .option('-t, --timeout <ms>', 'Request timeout (ms)', parseInt, 30000)
+    .option('-t, --timeout <ms>', 'Request timeout (ms)', (v) => parseInt(v, 10), 30000)
     .option('--ua <agent>', 'Custom user agent')
     .option('-s, --silent', 'Silent mode (no spinner)')
     .option('--screenshot [path]', 'Take a screenshot (optionally save to file path)')
@@ -142,7 +146,8 @@ program
     .option('--only-main-content', 'Shortcut for --include-tags main,article')
     .option('-H, --header <header...>', 'Custom headers (e.g., "Authorization: Bearer token")')
     .option('--cookie <cookie...>', 'Cookies to set (e.g., "session=abc123")')
-    .option('--cache <ttl>', 'Cache results locally (e.g., "5m", "1h", "1d")')
+    .option('--cache <ttl>', 'Cache results locally (e.g., "5m", "1h", "1d") — default: 5m')
+    .option('--no-cache', 'Disable automatic caching for this request')
     .option('--links', 'Output only the links found on the page')
     .option('--images', 'Output image URLs from the page')
     .option('--meta', 'Output only the page metadata (title, description, author, etc.)')
@@ -155,57 +160,112 @@ program
     .option('--location <country>', 'ISO country code for geo-targeting (e.g., "US", "DE", "JP")')
     .option('--language <lang>', 'Language preference (e.g., "en", "de", "ja")')
     .option('--max-tokens <n>', 'Maximum token count for output (truncate if exceeded)', parseInt)
+    .option('--budget <n>', 'Smart token budget — distill content to fit within N tokens (heuristic, no LLM key needed)', parseInt)
+    .option('--extract-all', 'Auto-detect and extract repeated listing items (e.g., search results)')
+    .option('--scroll-extract [count]', 'Scroll page N times to load lazy content, then extract (implies --render)', (v) => parseInt(v, 10))
+    .option('--csv', 'Output extraction results as CSV')
+    .option('--table', 'Output extraction results as a formatted table')
+    .option('--pages <n>', 'Follow pagination "Next" links for N pages (max 10)', (v) => parseInt(v, 10))
+    .option('--profile <path>', 'Use a persistent browser profile directory (cookies/sessions survive between calls)')
+    .option('--headed', 'Run browser in headed (visible) mode — useful for profile setup and debugging')
+    .option('--agent', 'Agent mode: sets --json, --silent, --extract-all, and --budget 4000 (override with --budget N)')
     .action(async (url, options) => {
-    if (!url) {
-        console.error('Error: URL is required\n');
-        program.help();
+    // --agent sets sensible defaults for AI agents; explicit flags override
+    if (options.agent) {
+        if (!options.json)
+            options.json = true;
+        if (!options.silent)
+            options.silent = true;
+        if (!options.extractAll)
+            options.extractAll = true;
+        if (options.budget === undefined)
+            options.budget = 4000;
+    }
+    const isJson = options.json;
+    // --- #5: Concise error for missing URL (no help dump) ---
+    if (!url || url.trim() === '') {
+        if (isJson) {
+            await writeStdout(JSON.stringify({ error: 'URL is required', code: 'URL_REQUIRED' }) + '\n');
+        }
+        else {
+            console.error('Error: URL is required');
+            console.error('Usage: webpeel <url> [options]');
+            console.error('Run "webpeel --help" for full usage.');
+        }
+        process.exit(1);
+    }
+    // --- #6: Helper to output JSON errors and exit ---
+    function exitWithJsonError(message, code) {
+        if (isJson) {
+            process.stdout.write(JSON.stringify({ error: message, code }) + '\n');
+        }
+        else {
+            console.error(`Error: ${message}`);
+        }
         process.exit(1);
     }
     // SECURITY: Enhanced URL validation
     if (url.length > 2048) {
-        console.error('Error: URL too long (max 2048 characters)');
-        process.exit(1);
+        exitWithJsonError('URL too long (max 2048 characters)', 'INVALID_URL');
     }
     // Check for control characters
     if (/[\x00-\x1F\x7F]/.test(url)) {
-        console.error('Error: URL contains invalid control characters');
-        process.exit(1);
+        exitWithJsonError('URL contains invalid control characters', 'INVALID_URL');
     }
     // Validate URL format
     try {
         const parsed = new URL(url);
         if (!['http:', 'https:'].includes(parsed.protocol)) {
-            console.error('Error: Only HTTP and HTTPS protocols are allowed');
-            process.exit(1);
+            exitWithJsonError('Only HTTP and HTTPS protocols are allowed', 'INVALID_URL');
         }
     }
     catch {
-        console.error(`Error: Invalid URL format: ${url}`);
-        process.exit(1);
+        exitWithJsonError(`Invalid URL format: ${url}`, 'INVALID_URL');
     }
     const useStealth = options.stealth || false;
     // Check usage quota
     const usageCheck = await checkUsage();
     if (!usageCheck.allowed) {
+        if (isJson) {
+            await writeStdout(JSON.stringify({ error: usageCheck.message, code: 'BLOCKED' }) + '\n');
+            process.exit(1);
+        }
         console.error(usageCheck.message);
         process.exit(1);
     }
     // Check cache first (before spinner/network)
+    // Default: 5m TTL for all CLI fetches unless --no-cache is set
     let cacheTtlMs;
-    if (options.cache) {
+    const cacheDisabled = options.cache === false; // --no-cache sets options.cache to false
+    const explicitTtl = typeof options.cache === 'string' ? options.cache : undefined;
+    if (!cacheDisabled) {
+        const ttlStr = explicitTtl || '5m';
         try {
-            cacheTtlMs = parseTTL(options.cache);
+            cacheTtlMs = parseTTL(ttlStr);
         }
         catch (e) {
-            console.error(`Error: ${e.message}`);
-            process.exit(1);
+            exitWithJsonError(e.message, 'FETCH_FAILED');
         }
-        const cached = getCache(url, { render: options.render, stealth: options.stealth, selector: options.selector, format: options.html ? 'html' : options.text ? 'text' : 'markdown' });
-        if (cached) {
+        const cacheOptions = {
+            render: options.render,
+            stealth: options.stealth,
+            selector: options.selector,
+            format: options.html ? 'html' : options.text ? 'text' : 'markdown',
+            budget: null, // Budget excluded from cache key — cache stores full content
+        };
+        const cachedResult = getCache(url, cacheOptions);
+        if (cachedResult) {
             if (!options.silent) {
-                console.error(`\x1b[36m⚡ Cache hit\x1b[0m (TTL: ${options.cache})`);
+                console.error(`\x1b[36m⚡ Cache hit\x1b[0m (TTL: ${ttlStr})`);
+            }
+            // Apply budget to cached content (cache stores full, budget is post-process)
+            if (options.budget && options.budget > 0 && cachedResult.content) {
+                const { distillToBudget } = await import('./core/budget.js');
+                const fmt = options.text ? 'text' : 'markdown';
+                cachedResult.content = distillToBudget(cachedResult.content, options.budget, fmt);
+                cachedResult.tokens = Math.ceil(cachedResult.content.length / 4);
             }
-            outputResult(cached, options);
+            await outputResult(cachedResult, options, { cached: true });
             process.exit(0);
         }
     }
@@ -213,8 +273,7 @@ program
     try {
         // Validate options
         if (options.wait && (options.wait < 0 || options.wait > 60000)) {
-            console.error('Error: Wait time must be between 0 and 60000ms');
-            process.exit(1);
+            throw Object.assign(new Error('Wait time must be between 0 and 60000ms'), { _code: 'FETCH_FAILED' });
         }
         // Parse custom headers
         let headers;
@@ -223,9 +282,7 @@ program
             for (const header of options.header) {
                 const colonIndex = header.indexOf(':');
                 if (colonIndex === -1) {
-                    console.error(`Error: Invalid header format: ${header}`);
-                    console.error('Expected format: "Key: Value"');
-                    process.exit(1);
+                    throw Object.assign(new Error(`Invalid header format: ${header}. Expected "Key: Value"`), { _code: 'FETCH_FAILED' });
                 }
                 const key = header.slice(0, colonIndex).trim();
                 const value = header.slice(colonIndex + 1).trim();
@@ -239,8 +296,7 @@ program
                 actions = parseActions(options.action);
             }
             catch (e) {
-                console.error(`Error: ${e.message}`);
-                process.exit(1);
+                throw Object.assign(new Error(e.message), { _code: 'FETCH_FAILED' });
             }
         }
         // Parse extract
@@ -254,8 +310,7 @@ program
                 llmBaseUrl: process.env.WEBPEEL_LLM_BASE_URL || 'https://api.openai.com/v1',
             };
             if (!extract.llmApiKey) {
-                console.error('Error: --llm-extract requires OPENAI_API_KEY environment variable');
-                process.exit(1);
+                throw Object.assign(new Error('--llm-extract requires OPENAI_API_KEY environment variable'), { _code: 'FETCH_FAILED' });
             }
         }
         else if (options.extract) {
@@ -264,15 +319,13 @@ program
                 extract = { selectors: JSON.parse(options.extract) };
             }
             catch {
-                console.error('Error: --extract must be valid JSON (e.g., \'{"title": "h1", "price": ".price"}\')');
-                process.exit(1);
+                throw Object.assign(new Error('--extract must be valid JSON (e.g., \'{"title": "h1", "price": ".price"}\')'), { _code: 'FETCH_FAILED' });
             }
         }
         // Validate maxTokens
         if (options.maxTokens !== undefined) {
             if (isNaN(options.maxTokens) || options.maxTokens < 100) {
-                console.error('Error: --max-tokens must be at least 100');
-                process.exit(1);
+                throw Object.assign(new Error('--max-tokens must be at least 100'), { _code: 'FETCH_FAILED' });
             }
         }
         // Parse include-tags and exclude-tags
@@ -301,7 +354,20 @@ program
         // Build peel options
         // --stealth auto-enables --render (stealth requires browser)
         // --action auto-enables --render (actions require browser)
-        const useRender = options.render || options.stealth || (actions && actions.length > 0) || false;
+        // --scroll-extract implies --render (needs browser)
+        const scrollExtractCount = options.scrollExtract !== undefined
+            ? (typeof options.scrollExtract === 'number' ? options.scrollExtract : 3)
+            : 0;
+        const useRender = options.render || options.stealth || (actions && actions.length > 0) || scrollExtractCount > 0 || false;
+        // Inject scroll actions when --scroll-extract is used
+        if (scrollExtractCount > 0) {
+            const scrollActions = [];
+            for (let i = 0; i < scrollExtractCount; i++) {
+                scrollActions.push({ type: 'scroll', to: 'bottom' });
+                scrollActions.push({ type: 'wait', ms: 1500 });
+            }
+            actions = actions ? [...actions, ...scrollActions] : scrollActions;
+        }
         const peelOptions = {
             render: useRender,
             stealth: options.stealth || false,
@@ -319,16 +385,20 @@ program
             raw: options.raw || false,
             actions,
             maxTokens: options.maxTokens,
+            // Note: budget is applied AFTER caching (so cache stores full content)
+            // We pass it to peel() for programmatic API compatibility, but the CLI
+            // also applies it post-fetch (see below) to ensure cache stores full result.
             extract,
             images: options.images || false,
             location: locationOptions,
+            profileDir: options.profile || undefined,
+            headed: options.headed || false,
         };
         // Add summary option if requested
         if (options.summary) {
             const llmApiKey = options.llmKey || process.env.OPENAI_API_KEY;
             if (!llmApiKey) {
-                console.error('Error: --summary requires --llm-key or OPENAI_API_KEY environment variable');
-                process.exit(1);
+                throw Object.assign(new Error('--summary requires --llm-key or OPENAI_API_KEY environment variable'), { _code: 'FETCH_FAILED' });
             }
             peelOptions.summary = true;
             peelOptions.llm = {
@@ -371,12 +441,162 @@ program
                 delete result.screenshot;
             }
         }
-        // Store in cache if caching is enabled
-        if (cacheTtlMs) {
-            setCache(url, result, cacheTtlMs, { render: options.render, stealth: useStealth, selector: options.selector, format: peelOptions.format });
+        // Store full result in cache (before budget distillation so cache is reusable)
+        if (cacheTtlMs && !cacheDisabled) {
+            setCache(url, result, cacheTtlMs, {
+                render: options.render,
+                stealth: useStealth,
+                selector: options.selector,
+                format: peelOptions.format,
+                budget: null, // Budget excluded — cache stores full content, budget applied post-cache
+            });
+        }
+        // Apply smart budget distillation AFTER caching (cache always stores full content)
+        // When --agent is set, always apply budget even with --extract-all (listings will be budgeted
+        // separately, but if no listings are found the content itself still needs trimming).
+        const skipBudgetForExtract = (options.extractAll || options.scrollExtract !== undefined) && !options.agent;
+        let contentTruncated = false;
+        if (options.budget && options.budget > 0 && !skipBudgetForExtract) {
+            const budgetFormat = peelOptions.format === 'text' ? 'text' : 'markdown';
+            const distilled = distillToBudget(result.content, options.budget, budgetFormat);
+            if (distilled !== result.content) {
+                contentTruncated = true;
+                result.content = distilled;
+                result.tokens = estimateTokens(distilled);
+            }
+        }
+        // --- #4: Content quality warning ---
+        const isHtmlContent = result.contentType ? result.contentType.toLowerCase().includes('html') : true;
+        const isRedirect = false; // peel() follows redirects — final result is always 200
+        if (result.tokens < 20 && !useRender && isHtmlContent && !isRedirect) {
+            const warningMsg = `Low content detected (${result.tokens} tokens). Try: webpeel ${url} --render`;
+            if (isJson) {
+                result.warning = warningMsg;
+            }
+            else {
+                console.error(`⚠ ${warningMsg}`);
+            }
+        }
+        // --- Extract-all / pagination / output formatting ---
+        const wantsExtractAll = options.extractAll || options.scrollExtract !== undefined;
+        const pagesCount = Math.min(Math.max(options.pages || 1, 1), 10);
+        if (wantsExtractAll) {
+            const { extractListings } = await import('./core/extract-listings.js');
+            const { findNextPageUrl } = await import('./core/paginate.js');
+            // We need the raw HTML for extraction. Re-fetch with format=html if needed.
+            let allListings = [];
+            // Fetch HTML for extraction
+            const htmlResult = peelOptions.format === 'html'
+                ? result
+                : await peel(url, { ...peelOptions, format: 'html', maxTokens: undefined });
+            allListings.push(...extractListings(htmlResult.content, result.url));
+            // Pagination: follow "Next" links
+            if (pagesCount > 1) {
+                let currentHtml = htmlResult.content;
+                let currentUrl = result.url;
+                for (let page = 1; page < pagesCount; page++) {
+                    const nextUrl = findNextPageUrl(currentHtml, currentUrl);
+                    if (!nextUrl)
+                        break;
+                    try {
+                        const nextResult = await peel(nextUrl, { ...peelOptions, format: 'html', maxTokens: undefined });
+                        const pageListings = extractListings(nextResult.content, nextResult.url);
+                        allListings.push(...pageListings);
+                        currentHtml = nextResult.content;
+                        currentUrl = nextResult.url;
+                    }
+                    catch {
+                        break; // Stop paginating on error
+                    }
+                }
+            }
+            // Apply budget to listings if requested
+            let listingsTruncated = false;
+            let totalAvailableListings;
+            if (options.budget && options.budget > 0 && allListings.length > 0) {
+                const { maxItems, truncated, totalAvailable } = budgetListings(allListings.length, options.budget);
+                if (truncated) {
+                    listingsTruncated = true;
+                    totalAvailableListings = totalAvailable;
+                    allListings = allListings.slice(0, maxItems);
+                }
+            }
+            // Output based on format flags
+            if (options.csv) {
+                const csvOutput = formatListingsCsv(allListings);
+                await writeStdout(csvOutput);
+            }
+            else if (options.table) {
+                const { formatTable } = await import('./core/table-format.js');
+                const tableRows = allListings.map(item => {
+                    const row = {};
+                    for (const [k, v] of Object.entries(item)) {
+                        if (v !== undefined)
+                            row[k] = v;
+                    }
+                    return row;
+                });
+                await writeStdout(formatTable(tableRows) + '\n');
+            }
+            else if (isJson) {
+                // Use unified envelope for JSON output
+                const structured = allListings;
+                const envelope = buildEnvelope(result, {
+                    cached: false,
+                    structured,
+                    truncated: listingsTruncated || undefined,
+                    totalAvailable: totalAvailableListings,
+                });
+                // Also include legacy fields for backward compat
+                envelope.listings = allListings;
+                envelope.count = allListings.length;
+                await writeStdout(JSON.stringify(envelope, null, 2) + '\n');
+            }
+            else {
+                // Formatted text output
+                if (allListings.length === 0) {
+                    await writeStdout('No listings found.\n');
+                }
+                else {
+                    const truncNote = listingsTruncated && totalAvailableListings
+                        ? ` (${totalAvailableListings} total — budget limited to ${allListings.length})`
+                        : '';
+                    await writeStdout(`Found ${allListings.length} listings${truncNote}:\n\n`);
+                    allListings.forEach((item, i) => {
+                        const pricePart = item.price ? ` — ${item.price}` : '';
+                        const line = `${i + 1}. ${item.title}${pricePart}\n`;
+                        process.stdout.write(line);
+                        if (item.link) {
+                            process.stdout.write(`   ${item.link}\n`);
+                        }
+                        process.stdout.write('\n');
+                    });
+                }
+            }
+        }
+        else if (options.csv || options.table) {
+            // CSV / table output for --extract (CSS selector extraction)
+            if (result.extracted) {
+                const rows = normaliseExtractedToRows(result.extracted);
+                if (options.csv) {
+                    await writeStdout(formatListingsCsv(rows));
+                }
+                else {
+                    const { formatTable } = await import('./core/table-format.js');
+                    await writeStdout(formatTable(rows) + '\n');
+                }
+            }
+            else {
+                console.error('--csv / --table require --extract-all or --extract to produce structured data.');
+            }
+        }
+        else {
+            // Output results (default path)
+            await outputResult(result, options, {
+                cached: false,
+                truncated: contentTruncated || undefined,
+            });
         }
-        // Output results
-        await outputResult(result, options);
         // Clean up and exit
         await cleanup();
         process.exit(0);
@@ -385,6 +605,14 @@ program
         if (spinner) {
             spinner.fail('Failed to fetch');
         }
+        // --- #6: Consistent JSON error output ---
+        if (isJson) {
+            const errMsg = error instanceof Error ? error.message : 'Unknown error';
+            const errCode = classifyErrorCode(error);
+            await writeStdout(JSON.stringify({ error: errMsg, code: errCode }) + '\n');
+            await cleanup();
+            process.exit(1);
+        }
         if (error instanceof Error) {
             console.error(`\nError: ${error.message}`);
             // Provide actionable hints based on error type
@@ -418,22 +646,122 @@ program
 // Search command
 program
     .command('search <query>')
-    .description('Search the web (DuckDuckGo by default, or Brave with --provider brave)')
+    .description('Search the web (DuckDuckGo by default, or use --site for site-specific search)')
     .option('-n, --count <n>', 'Number of results (1-10)', '5')
+    .option('--top <n>', 'Limit results (alias for --count)')
     .option('--provider <provider>', 'Search provider: duckduckgo (default) or brave')
     .option('--search-api-key <key>', 'API key for the search provider (or env WEBPEEL_BRAVE_API_KEY)')
+    .option('--site <site>', 'Search a specific site (e.g. ebay, amazon, github). Run "webpeel sites" for full list.')
     .option('--json', 'Output as JSON')
+    .option('--urls-only', 'Output only URLs, one per line (pipe-friendly)')
+    .option('--table', 'Output site-search results as a formatted table (requires --site)')
+    .option('--csv', 'Output site-search results as CSV (requires --site)')
+    .option('--budget <n>', 'Token budget for site-search result content', parseInt)
     .option('-s, --silent', 'Silent mode')
     .action(async (query, options) => {
     const isJson = options.json;
     const isSilent = options.silent;
-    const count = parseInt(options.count) || 5;
+    // --top overrides --count when both are provided
+    const count = parseInt(options.top ?? options.count) || 5;
     // Check usage quota
     const usageCheck = await checkUsage();
     if (!usageCheck.allowed) {
         console.error(usageCheck.message);
         process.exit(1);
     }
+    // ── --site: site-specific structured search ───────────────────────────
+    if (options.site) {
+        const spinner = isSilent ? null : ora(`Searching ${options.site}...`).start();
+        try {
+            const { buildSiteSearchUrl } = await import('./core/site-search.js');
+            const siteResult = buildSiteSearchUrl(options.site, query);
+            // Fetch the raw HTML (needed for listing extraction)
+            const htmlResult = await peel(siteResult.url, {
+                format: 'html',
+                timeout: 30000,
+            });
+            if (spinner) {
+                spinner.succeed(`Fetched ${siteResult.site} in ${htmlResult.elapsed}ms`);
+            }
+            // Extract listings from the HTML
+            const { extractListings } = await import('./core/extract-listings.js');
+            let listings = extractListings(htmlResult.content, siteResult.url);
+            // Apply budget if requested
+            if (options.budget && options.budget > 0 && listings.length > 0) {
+                const { budgetListings } = await import('./core/budget.js');
+                const { maxItems } = budgetListings(listings.length, options.budget);
+                listings = listings.slice(0, maxItems);
+            }
+            // Show usage footer
+            if (usageCheck.usageInfo && !isSilent) {
+                showUsageFooter(usageCheck.usageInfo, usageCheck.isAnonymous || false, false);
+            }
+            // Output
+            if (options.csv) {
+                const rows = listings.map(item => {
+                    const row = {};
+                    for (const [k, v] of Object.entries(item)) {
+                        if (v !== undefined)
+                            row[k] = v;
+                    }
+                    return row;
+                });
+                await writeStdout(formatListingsCsv(rows));
+            }
+            else if (options.table) {
+                const { formatTable } = await import('./core/table-format.js');
+                const rows = listings.map(item => {
+                    const row = {};
+                    for (const [k, v] of Object.entries(item)) {
+                        if (v !== undefined)
+                            row[k] = v;
+                    }
+                    return row;
+                });
+                await writeStdout(formatTable(rows) + '\n');
+            }
+            else if (isJson) {
+                const envelope = {
+                    site: siteResult.site,
+                    query: siteResult.query,
+                    url: siteResult.url,
+                    count: listings.length,
+                    items: listings,
+                    elapsed: htmlResult.elapsed,
+                };
+                await writeStdout(JSON.stringify(envelope, null, 2) + '\n');
+            }
+            else {
+                if (listings.length === 0) {
+                    await writeStdout('No listings found.\n');
+                }
+                else {
+                    await writeStdout(`Found ${listings.length} listings on ${siteResult.site}:\n\n`);
+                    for (const [i, item] of listings.entries()) {
+                        const pricePart = item.price ? ` — ${item.price}` : '';
+                        process.stdout.write(`${i + 1}. ${item.title}${pricePart}\n`);
+                        if (item.link)
+                            process.stdout.write(`   ${item.link}\n`);
+                        process.stdout.write('\n');
+                    }
+                }
+            }
+            await cleanup();
+            process.exit(0);
+        }
+        catch (error) {
+            if (spinner)
+                spinner.fail('Site search failed');
+            if (error instanceof Error) {
+                console.error(`\nError: ${error.message}`);
+            }
+            else {
+                console.error('\nError: Unknown error occurred');
+            }
+            await cleanup();
+            process.exit(1);
+        }
+    }
     const spinner = isSilent ? null : ora('Searching...').start();
     try {
         const { getSearchProvider } = await import('./core/search-provider.js');
@@ -456,16 +784,15 @@ program
         if (usageCheck.usageInfo && !isSilent) {
             showUsageFooter(usageCheck.usageInfo, usageCheck.isAnonymous || false, false);
         }
-        if (isJson) {
+        if (options.urlsOnly) {
+            // Pipe-friendly: one URL per line
+            for (const result of results) {
+                await writeStdout(result.url + '\n');
+            }
+        }
+        else if (isJson) {
             const jsonStr = JSON.stringify(results, null, 2);
-            await new Promise((resolve, reject) => {
-                process.stdout.write(jsonStr + '\n', (err) => {
-                    if (err)
-                        reject(err);
-                    else
-                        resolve();
-                });
-            });
+            await writeStdout(jsonStr + '\n');
         }
         else {
             for (const result of results) {
@@ -497,6 +824,44 @@ program
         process.exit(1);
     }
 });
+// Sites command — list all supported site templates
+program
+    .command('sites')
+    .description('List all sites supported by "webpeel search --site <site>"')
+    .option('--json', 'Output as JSON')
+    .option('--category <cat>', 'Filter by category (shopping, social, tech, jobs, general, real-estate, food)')
+    .action(async (options) => {
+    const { listSites } = await import('./core/site-search.js');
+    let sites = listSites();
+    if (options.category) {
+        sites = sites.filter(s => s.category === options.category);
+    }
+    if (options.json) {
+        await writeStdout(JSON.stringify(sites, null, 2) + '\n');
+        process.exit(0);
+    }
+    // Group by category for pretty output
+    const byCategory = new Map();
+    for (const site of sites) {
+        if (!byCategory.has(site.category))
+            byCategory.set(site.category, []);
+        byCategory.get(site.category).push(site);
+    }
+    const categoryOrder = ['shopping', 'general', 'social', 'tech', 'jobs', 'real-estate', 'food'];
+    const sortedCategories = categoryOrder.filter(c => byCategory.has(c));
+    console.log('\nWebPeel Site-Aware Search — supported sites\n');
+    console.log('Usage: webpeel search --site <id> "<query>"\n');
+    for (const cat of sortedCategories) {
+        const catSites = byCategory.get(cat);
+        const label = cat.charAt(0).toUpperCase() + cat.slice(1);
+        console.log(`  ${label}:`);
+        for (const s of catSites) {
+            console.log(`    ${s.id.padEnd(16)} ${s.name}`);
+        }
+        console.log('');
+    }
+    process.exit(0);
+});
 // Batch command
 program
     .command('batch [file]')
@@ -632,12 +997,12 @@ program
 program
     .command('crawl <url>')
     .description('Crawl a website starting from a URL')
-    .option('--max-pages <number>', 'Maximum number of pages to crawl (default: 10, max: 100)', parseInt, 10)
-    .option('--max-depth <number>', 'Maximum depth to crawl (default: 2, max: 5)', parseInt, 2)
+    .option('--max-pages <number>', 'Maximum number of pages to crawl (default: 10, max: 100)', (v) => parseInt(v, 10), 10)
+    .option('--max-depth <number>', 'Maximum depth to crawl (default: 2, max: 5)', (v) => parseInt(v, 10), 2)
     .option('--allowed-domains <domains...>', 'Only crawl these domains (default: same as starting URL)')
     .option('--exclude <patterns...>', 'Exclude URLs matching these regex patterns')
     .option('--ignore-robots', 'Ignore robots.txt (default: respect robots.txt)')
-    .option('--rate-limit <ms>', 'Rate limit between requests in ms (default: 1000)', parseInt, 1000)
+    .option('--rate-limit <ms>', 'Rate limit between requests in ms (default: 1000)', (v) => parseInt(v, 10), 1000)
     .option('-r, --render', 'Use headless browser for all pages')
     .option('--stealth', 'Use stealth mode for all pages')
     .option('-s, --silent', 'Silent mode (no spinner)')
@@ -710,7 +1075,7 @@ program
     .description('Discover all URLs on a domain (sitemap + crawl)')
     .option('--no-sitemap', 'Skip sitemap.xml discovery')
     .option('--no-crawl', 'Skip homepage crawl')
-    .option('--max <n>', 'Maximum URLs to discover (default: 5000)', parseInt, 5000)
+    .option('--max <n>', 'Maximum URLs to discover (default: 5000)', (v) => parseInt(v, 10), 5000)
     .option('--include <patterns...>', 'Include only URLs matching these regex patterns')
     .option('--exclude <patterns...>', 'Exclude URLs matching these regex patterns')
     .option('--json', 'Output as JSON')
@@ -751,6 +1116,177 @@ program
         process.exit(1);
     }
 });
+// Watch command - monitor a URL for changes / assertion failures
+program
+    .command('watch <url>')
+    .description('Monitor a URL for changes and assertion failures')
+    .option('--interval <duration>', 'Check interval (e.g. 30s, 5m, 1h)', '5m')
+    .option('--assert <condition...>', 'Assertion(s) to check (e.g. "status=200" "body.health=ok")')
+    .option('--webhook <url>', 'POST this URL on assertion failure or content change')
+    .option('-t, --timeout <ms>', 'Per-request timeout in ms', (v) => parseInt(v, 10), 10000)
+    .option('--max-checks <n>', 'Stop after N checks (default: unlimited)', (v) => parseInt(v, 10))
+    .option('--json', 'Output each check as NDJSON to stdout')
+    .option('-s, --silent', 'Only output on failures/changes')
+    .option('-r, --render', 'Use browser rendering for checks')
+    .action(async (url, options) => {
+    const { watch: runWatch, parseDuration, parseAssertion } = await import('./core/watch.js');
+    // Validate URL
+    try {
+        const parsed = new URL(url);
+        if (!['http:', 'https:'].includes(parsed.protocol)) {
+            console.error('Error: Only HTTP and HTTPS protocols are allowed');
+            process.exit(1);
+        }
+    }
+    catch {
+        console.error(`Error: Invalid URL format: ${url}`);
+        process.exit(1);
+    }
+    // Parse interval
+    let intervalMs;
+    try {
+        intervalMs = parseDuration(options.interval);
+    }
+    catch (e) {
+        console.error(`Error: ${e.message}`);
+        process.exit(1);
+    }
+    // Parse assertions
+    const assertions = [];
+    if (options.assert && Array.isArray(options.assert)) {
+        for (const expr of options.assert) {
+            try {
+                assertions.push(parseAssertion(expr));
+            }
+            catch (e) {
+                console.error(`Error: ${e.message}`);
+                process.exit(1);
+            }
+        }
+    }
+    if (!options.json && !options.silent) {
+        const intervalLabel = options.interval;
+        const assertLabel = assertions.length > 0
+            ? ` with ${assertions.length} assertion(s)`
+            : '';
+        process.stderr.write(`Watching ${url} every ${intervalLabel}${assertLabel}. Press Ctrl+C to stop.\n`);
+    }
+    const watchOptions = {
+        url,
+        intervalMs,
+        assertions,
+        webhookUrl: options.webhook,
+        timeout: options.timeout,
+        maxChecks: options.maxChecks,
+        render: options.render || false,
+        json: options.json || false,
+        silent: options.silent || false,
+    };
+    try {
+        await runWatch(watchOptions);
+    }
+    catch (error) {
+        console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        process.exit(1);
+    }
+    process.exit(0);
+});
+// Diff command - semantic diff against last snapshot
+program
+    .command('diff <url>')
+    .description('Show semantic diff between current content and the last tracked snapshot')
+    .option('--last', 'Compare against last tracked snapshot (default)')
+    .option('--against <snapshot-url>', 'Compare against the snapshot stored for a different URL')
+    .option('--fields <fields>', 'For JSON responses: only diff these fields (comma-separated dot-notation)')
+    .option('--json', 'Output diff as JSON')
+    .option('-r, --render', 'Use browser rendering')
+    .option('-t, --timeout <ms>', 'Request timeout in ms', (v) => parseInt(v, 10), 30000)
+    .option('-s, --silent', 'Silent mode (no spinner)')
+    .action(async (url, options) => {
+    const isJson = options.json;
+    // Validate URL
+    try {
+        const parsed = new URL(url);
+        if (!['http:', 'https:'].includes(parsed.protocol)) {
+            if (isJson) {
+                await writeStdout(JSON.stringify({ error: 'Only HTTP and HTTPS protocols are allowed', code: 'INVALID_URL' }) + '\n');
+            }
+            else {
+                console.error('Error: Only HTTP and HTTPS protocols are allowed');
+            }
+            process.exit(1);
+        }
+    }
+    catch {
+        if (isJson) {
+            await writeStdout(JSON.stringify({ error: `Invalid URL format: ${url}`, code: 'INVALID_URL' }) + '\n');
+        }
+        else {
+            console.error(`Error: Invalid URL format: ${url}`);
+        }
+        process.exit(1);
+    }
+    const spinner = options.silent ? null : ora('Fetching and diffing...').start();
+    try {
+        const { diffUrl } = await import('./core/diff.js');
+        const fields = options.fields
+            ? options.fields.split(',').map((f) => f.trim()).filter(Boolean)
+            : undefined;
+        const result = await diffUrl(url, {
+            render: options.render || false,
+            timeout: options.timeout,
+            fields,
+        });
+        if (spinner) {
+            spinner.succeed(`Diff completed in ${result.changed ? 'CHANGED' : 'no change'}`);
+        }
+        if (isJson) {
+            await writeStdout(JSON.stringify(result, null, 2) + '\n');
+        }
+        else {
+            // Human-readable output
+            const ago = result.previousTimestamp
+                ? formatRelativeTime(new Date(result.previousTimestamp))
+                : 'unknown';
+            console.log(`\nComparing ${result.url} (now vs ${ago})\n`);
+            if (!result.changed) {
+                console.log('  No changes detected.');
+            }
+            else {
+                for (const change of result.changes) {
+                    const label = change.field ?? change.path ?? '(unknown)';
+                    if (change.type === 'modified') {
+                        console.log(`  Modified: ${label}  ${change.before} → ${change.after}`);
+                    }
+                    else if (change.type === 'added') {
+                        console.log(`  Added:    ${label}  ${change.after}`);
+                    }
+                    else if (change.type === 'removed') {
+                        console.log(`  Removed:  ${label}  ${change.before}`);
+                    }
+                }
+            }
+            console.log(`\nSummary: ${result.summary}`);
+        }
+        await cleanup();
+        process.exit(0);
+    }
+    catch (error) {
+        if (spinner)
+            spinner.fail('Diff failed');
+        if (isJson) {
+            await writeStdout(JSON.stringify({
+                error: error instanceof Error ? error.message : 'Unknown error',
+                code: 'FETCH_FAILED',
+            }) + '\n');
+        }
+        else {
+            console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+        await cleanup();
+        process.exit(1);
+    }
+});
 program
     .command('login')
     .description('Authenticate the CLI with your API key')
@@ -987,33 +1523,46 @@ program
 // Track command - track changes on a URL
 program
     .command('track <url>')
-    .description('Track changes on a URL (returns fingerprint for change detection)')
+    .description('Track changes on a URL (saves snapshot for use with `webpeel diff`)')
     .option('-s, --silent', 'Silent mode (no spinner)')
     .option('--json', 'Output as JSON')
+    .option('-r, --render', 'Use browser rendering')
     .action(async (url, options) => {
     const spinner = options.silent ? null : ora('Fetching and tracking...').start();
     try {
-        const result = await peel(url);
+        // changeTracking: true saves the snapshot to ~/.webpeel/snapshots/ so that
+        // `webpeel diff` can compare against it later.
+        const result = await peel(url, {
+            render: options.render || false,
+            changeTracking: true,
+        });
         if (spinner) {
             spinner.succeed(`Tracked in ${result.elapsed}ms`);
         }
+        const changeStatus = result.changeTracking?.changeStatus ?? 'new';
+        const previousScrapeAt = result.changeTracking?.previousScrapeAt ?? null;
         if (options.json) {
-            console.log(JSON.stringify({
+            await writeStdout(JSON.stringify({
                 url: result.url,
                 title: result.title,
                 fingerprint: result.fingerprint,
                 tokens: result.tokens,
                 contentType: result.contentType,
+                changeStatus,
+                previousScrapeAt,
                 lastChecked: new Date().toISOString(),
-            }, null, 2));
+            }, null, 2) + '\n');
         }
         else {
             console.log(`URL: ${result.url}`);
             console.log(`Title: ${result.title}`);
             console.log(`Fingerprint: ${result.fingerprint}`);
             console.log(`Tokens: ${result.tokens}`);
+            console.log(`Status: ${changeStatus}`);
+            if (previousScrapeAt)
+                console.log(`Previous check: ${previousScrapeAt}`);
             console.log(`Last checked: ${new Date().toISOString()}`);
-            console.log('\nSave this fingerprint to detect future changes.');
+            console.log('\nSnapshot saved. Run `webpeel diff <url> --last` to compare future changes.');
         }
         await cleanup();
         process.exit(0);
@@ -1145,25 +1694,39 @@ program
         process.exit(1);
     }
 });
-// Jobs command - search job boards (LinkedIn, Indeed, Glassdoor)
-program
-    .command('jobs <keywords>')
-    .description('Search job boards for listings (LinkedIn, Indeed, Glassdoor)')
+// ── Jobs command group ─────────────────────────────────────────────────────
+const jobsCmd = program
+    .command('jobs')
+    .description('Job board operations: search listings and auto-apply (LinkedIn, Indeed, Glassdoor, Upwork)')
+    .argument('[keywords]', 'Search keywords — shorthand for "jobs search <keywords>"')
     .option('-l, --location <location>', 'Location filter')
-    .option('-s, --source <source>', 'Job board: glassdoor, indeed, or linkedin (default: linkedin)', 'linkedin')
+    .option('-s, --source <source>', 'Job board: glassdoor, indeed, linkedin, or upwork (default: linkedin)', 'linkedin')
     .option('-n, --limit <number>', 'Max results (default: 25)', '25')
     .option('-d, --details <number>', 'Fetch full details for top N results (default: 0)', '0')
     .option('--json', 'Output raw JSON')
     .option('--timeout <ms>', 'Request timeout in ms (default: 30000)', '30000')
     .option('--silent', 'Silent mode (no spinner)')
     .action(async (keywords, options) => {
+    // Default action: when called as `webpeel jobs <keywords>`, act as search
+    if (!keywords) {
+        jobsCmd.help();
+        process.exit(0);
+    }
+    // Delegate to shared search logic
+    await runJobSearch(keywords, options);
+});
+// ── Shared job-search logic (used by both `jobs` default and `jobs search`) ───
+async function runJobSearch(keywords, options) {
     const spinner = options.silent ? null : ora('Searching jobs...').start();
     try {
         const { searchJobs } = await import('./core/jobs.js');
-        const source = (['glassdoor', 'indeed', 'linkedin'].includes(options.source) ? options.source : 'linkedin');
-        const limit = Math.min(Math.max(parseInt(options.limit, 10) || 25, 1), 100);
-        const fetchDetails = Math.min(Math.max(parseInt(options.details, 10) || 0, 0), limit);
-        const timeout = parseInt(options.timeout, 10) || 30000;
+        const VALID_SOURCES = ['glassdoor', 'indeed', 'linkedin', 'upwork'];
+        const source = (VALID_SOURCES.includes((options.source ?? 'linkedin'))
+            ? options.source
+            : 'linkedin');
+        const limit = Math.min(Math.max(parseInt(options.limit ?? '25', 10) || 25, 1), 100);
+        const fetchDetails = Math.min(Math.max(parseInt(options.details ?? '0', 10) || 0, 0), limit);
+        const timeout = parseInt(options.timeout ?? '30000', 10) || 30000;
         const result = await searchJobs({
             keywords,
             location: options.location,
@@ -1174,12 +1737,10 @@ program
         });
         if (spinner)
             spinner.stop();
-        // --json: raw output
         if (options.json) {
             await writeStdout(JSON.stringify(result, null, 2) + '\n');
             process.exit(0);
         }
-        // Formatted table output
         const totalLabel = result.totalFound >= 1000
             ? `${(result.totalFound / 1000).toFixed(0).replace(/\.0$/, '')}k+`
             : String(result.totalFound);
@@ -1189,7 +1750,6 @@ program
             console.log('  No jobs found.\n');
             process.exit(0);
         }
-        // Column widths
         const colNum = 3;
         const colTitle = 40;
         const colCompany = 18;
@@ -1198,18 +1758,15 @@ program
         const colPosted = 10;
         const pad = (s, w) => s.length > w ? s.slice(0, w - 1) + '…' : s.padEnd(w);
         const rpad = (s, w) => s.padStart(w);
-        // Header
-        console.log(` ${rpad('#', colNum)}  ${pad('Title', colTitle)}  ${pad('Company', colCompany)}  ${pad('Location', colLocation)}  ${pad('Salary', colSalary)}  ${pad('Posted', colPosted)}`);
-        // Rows
+        console.log(` ${rpad('#', colNum)}  ${pad('Title', colTitle)}  ${pad('Company', colCompany)}  ${pad('Location', colLocation)}  ${pad('Salary/Budget', colSalary)}  ${pad('Posted', colPosted)}`);
         result.jobs.forEach((job, i) => {
-            const title = job.title + (job.remote ? ' 🏠' : '');
-            console.log(` ${rpad(String(i + 1), colNum)}  ${pad(title, colTitle)}  ${pad(job.company, colCompany)}  ${pad(job.location, colLocation)}  ${pad(job.salary || '', colSalary)}  ${pad(job.postedAt || '', colPosted)}`);
+            const titleStr = job.title + (job.remote ? ' 🏠' : '');
+            const salaryStr = job.salary ?? ('budget' in job ? job.budget : '') ?? '';
+            console.log(` ${rpad(String(i + 1), colNum)}  ${pad(titleStr, colTitle)}  ${pad(job.company, colCompany)}  ${pad(job.location, colLocation)}  ${pad(salaryStr, colSalary)}  ${pad(job.postedAt ?? '', colPosted)}`);
         });
-        // Footer
         const timeSec = (result.timeTakenMs / 1000).toFixed(1);
         const detailsNote = fetchDetails > 0 ? ` | Details: ${result.detailsFetched} fetched` : '';
         console.log(`\nFetched ${result.jobs.length} jobs in ${timeSec}s${detailsNote}\n`);
-        // Detailed job cards (when --details > 0)
         const detailedJobs = result.jobs.filter((j) => 'description' in j);
         for (let i = 0; i < detailedJobs.length; i++) {
             const job = detailedJobs[i];
@@ -1251,7 +1808,238 @@ program
     }
     catch (error) {
         if (spinner)
-            spinner.fail('Job search failed');
+            spinner.fail?.('Job search failed');
+        console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        process.exit(1);
+    }
+}
+// jobs search <keywords>  — explicit subcommand (same logic as default action)
+jobsCmd
+    .command('search <keywords>')
+    .description('Search job boards for listings (LinkedIn, Indeed, Glassdoor, Upwork)')
+    .alias('s')
+    .option('-l, --location <location>', 'Location filter')
+    .option('-s, --source <source>', 'Job board: glassdoor, indeed, linkedin, or upwork (default: linkedin)', 'linkedin')
+    .option('-n, --limit <number>', 'Max results (default: 25)', '25')
+    .option('-d, --details <number>', 'Fetch full details for top N results (default: 0)', '0')
+    .option('--json', 'Output raw JSON')
+    .option('--timeout <ms>', 'Request timeout in ms (default: 30000)', '30000')
+    .option('--silent', 'Silent mode (no spinner)')
+    .action(async (keywords, options) => {
+    await runJobSearch(keywords, options);
+});
+// ── jobs apply <url>  ─────────────────────────────────────────────────────────
+// Stealth automated job application using human behavior simulation
+jobsCmd
+    .command('apply <url>')
+    .description('Stealth automated job application using human behavior simulation')
+    .option('--profile <path>', 'Path to profile JSON file', `${process.env.HOME ?? '~'}/.webpeel/profile.json`)
+    .option('--resume <path>', 'Path to resume PDF (overrides profile.resumePath)')
+    .option('--mode <mode>', 'Submission mode: auto | review | dry-run (default: review)', 'review')
+    .option('--session-dir <path>', 'Browser session directory (preserves login cookies)')
+    .option('--llm-key <key>', 'LLM API key for custom question answers')
+    .option('--llm-provider <name>', 'LLM provider: openai | anthropic (default: openai)', 'openai')
+    .option('--daily-limit <n>', 'Max applications per day (default: 8)', '8')
+    .option('--no-warmup', 'Skip browsing warmup phase')
+    .option('--json', 'Output result as JSON')
+    .option('--silent', 'Minimal output')
+    .action(async (url, options) => {
+    const isSilent = options.silent;
+    const isJson = options.json;
+    const mode = (['auto', 'review', 'dry-run'].includes(options.mode)
+        ? options.mode
+        : 'review');
+    if (!isSilent) {
+        console.log(`\n🤖 WebPeel Auto-Apply — mode: ${mode}`);
+        console.log(`   URL: ${url}\n`);
+    }
+    // Load profile
+    const profilePath = options.profile;
+    let profile;
+    try {
+        const raw = readFileSync(profilePath, 'utf-8');
+        profile = JSON.parse(raw);
+    }
+    catch {
+        console.error(`Error: Could not load profile from ${profilePath}`);
+        console.error(`Run "webpeel jobs apply-setup" to create a profile.`);
+        process.exit(1);
+    }
+    if (options.resume) {
+        profile.resumePath = options.resume;
+    }
+    const spinner = isSilent ? null : ora('Applying...').start();
+    try {
+        const { applyToJob } = await import('./core/apply.js');
+        const result = await applyToJob({
+            url,
+            profile,
+            mode,
+            sessionDir: options.sessionDir,
+            llmKey: options.llmKey,
+            llmProvider: options.llmProvider,
+            dailyLimit: parseInt(options.dailyLimit, 10) || 8,
+            warmup: options.warmup !== false,
+            onProgress: isSilent
+                ? undefined
+                : (event) => {
+                    if (spinner)
+                        spinner.text = `[${event.stage}] ${event.message}`;
+                    else
+                        console.log(`  [${event.stage}] ${event.message}`);
+                },
+        });
+        if (spinner)
+            spinner.stop();
+        if (isJson) {
+            await writeStdout(JSON.stringify(result, null, 2) + '\n');
+            process.exit(result.error ? 1 : 0);
+        }
+        const statusIcon = result.submitted ? '✅' : result.error ? '❌' : '📋';
+        console.log(`\n${statusIcon} ${result.submitted
+            ? 'Application submitted!'
+            : result.error
+                ? `Error: ${result.error}`
+                : 'Application completed (not submitted)'}`);
+        if (result.job.title || result.job.company) {
+            console.log(`   ${result.job.title}${result.job.company ? ` @ ${result.job.company}` : ''}`);
+        }
+        console.log(`\n   Fields filled: ${result.fieldsFilled}`);
+        if (result.llmAnswers > 0)
+            console.log(`   LLM answers: ${result.llmAnswers}`);
+        if (result.fieldsSkipped.length > 0)
+            console.log(`   Skipped: ${result.fieldsSkipped.join(', ')}`);
+        if (result.warnings.length > 0 && !isSilent) {
+            console.log(`\n   Warnings:`);
+            result.warnings.forEach(w => console.log(`   ⚠️  ${w}`));
+        }
+        console.log(`   Time: ${(result.elapsed / 1000).toFixed(1)}s\n`);
+        process.exit(result.error ? 1 : 0);
+    }
+    catch (error) {
+        if (spinner)
+            spinner.fail('Application failed');
+        console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        process.exit(1);
+    }
+});
+// ── jobs apply-setup  ─────────────────────────────────────────────────────────
+// Interactive wizard to create ~/.webpeel/profile.json
+jobsCmd
+    .command('apply-setup')
+    .description('Interactive setup wizard — creates ~/.webpeel/profile.json')
+    .action(async () => {
+    const { createInterface } = await import('readline');
+    const rl = createInterface({ input: process.stdin, output: process.stdout });
+    const ask = (q) => new Promise(resolve => rl.question(q, ans => resolve(ans.trim())));
+    console.log('\n🤖 WebPeel Apply Setup — Create your applicant profile\n');
+    console.log('This creates ~/.webpeel/profile.json used by "webpeel jobs apply".\n');
+    try {
+        const name = await ask('Full name: ');
+        const email = await ask('Email address: ');
+        const phone = await ask('Phone number: ');
+        const linkedin = await ask('LinkedIn URL (optional, press Enter to skip): ');
+        const website = await ask('Portfolio/website URL (optional): ');
+        const location = await ask('City, State (e.g. San Francisco, CA): ');
+        const workAuth = await ask('Work authorization (e.g. US Citizen, Permanent Resident, H-1B, Need Sponsorship): ');
+        const yearsExp = await ask('Years of experience: ');
+        const currentTitle = await ask('Current/most recent job title: ');
+        const skills = await ask('Skills (comma-separated, e.g. TypeScript, React, Node.js): ');
+        const education = await ask('Education (e.g. B.S. Computer Science, MIT): ');
+        const resumePath = await ask('Path to resume PDF (e.g. /Users/you/resume.pdf): ');
+        const summary = await ask('Professional summary (1-3 sentences): ');
+        const salaryMin = await ask('Minimum desired salary (optional, e.g. 120000): ');
+        const salaryMax = await ask('Maximum desired salary (optional, e.g. 180000): ');
+        const relocate = await ask('Willing to relocate? (y/n): ');
+        const sponsorship = await ask('Need visa sponsorship? (y/n): ');
+        rl.close();
+        const profileData = {
+            name,
+            email,
+            phone,
+            ...(linkedin ? { linkedin } : {}),
+            ...(website ? { website } : {}),
+            location,
+            workAuthorization: workAuth,
+            yearsExperience: parseInt(yearsExp, 10) || 0,
+            currentTitle,
+            skills: skills.split(',').map(s => s.trim()).filter(Boolean),
+            education,
+            resumePath,
+            summary,
+            ...(salaryMin && salaryMax
+                ? { salaryRange: { min: parseInt(salaryMin, 10), max: parseInt(salaryMax, 10) } }
+                : {}),
+            willingToRelocate: relocate.toLowerCase().startsWith('y'),
+            needsSponsorship: sponsorship.toLowerCase().startsWith('y'),
+        };
+        const { mkdirSync: mk, writeFileSync: wf, existsSync: ex } = await import('fs');
+        const { join: j } = await import('path');
+        const { homedir: hd } = await import('os');
+        const webpeelDir = j(hd(), '.webpeel');
+        if (!ex(webpeelDir))
+            mk(webpeelDir, { recursive: true });
+        const profilePath = j(webpeelDir, 'profile.json');
+        wf(profilePath, JSON.stringify(profileData, null, 2), 'utf-8');
+        console.log(`\n✅ Profile saved to: ${profilePath}`);
+        console.log('\nNext steps:');
+        console.log('  1. Apply to a job: webpeel jobs apply https://linkedin.com/jobs/view/...');
+        console.log('     (First run opens a browser — log in to LinkedIn, then the session is saved)\n');
+    }
+    catch (error) {
+        rl.close();
+        console.error(`\nError: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        process.exit(1);
+    }
+});
+// ── jobs apply-history  ───────────────────────────────────────────────────────
+// View application history from ~/.webpeel/applications.json
+jobsCmd
+    .command('apply-history')
+    .description('View application history from ~/.webpeel/applications.json')
+    .option('--json', 'Output as JSON')
+    .option('--limit <n>', 'Number of recent applications to show (default: 20)', '20')
+    .action(async (options) => {
+    const isJson = options.json;
+    const limit = parseInt(options.limit, 10) || 20;
+    try {
+        const { loadApplications } = await import('./core/apply.js');
+        const allApps = loadApplications();
+        const apps = allApps.slice().reverse().slice(0, limit);
+        if (isJson) {
+            await writeStdout(JSON.stringify(apps, null, 2) + '\n');
+            process.exit(0);
+        }
+        if (apps.length === 0) {
+            console.log('\nNo applications yet. Use "webpeel jobs apply <url>" to start.\n');
+            process.exit(0);
+        }
+        console.log(`\n📋 Application History (${apps.length} of ${allApps.length} total)\n`);
+        const colDate = 22;
+        const colStatus = 10;
+        const colTitle = 35;
+        const colCompany = 20;
+        const colMode = 8;
+        const pad = (s, w) => (s.length > w ? s.slice(0, w - 1) + '…' : s.padEnd(w));
+        console.log(` ${pad('Applied', colDate)}  ${pad('Status', colStatus)}  ${pad('Title', colTitle)}  ${pad('Company', colCompany)}  ${pad('Mode', colMode)}`);
+        console.log(` ${'-'.repeat(colDate)}  ${'-'.repeat(colStatus)}  ${'-'.repeat(colTitle)}  ${'-'.repeat(colCompany)}  ${'-'.repeat(colMode)}`);
+        for (const app of apps) {
+            const date = new Date(app.appliedAt).toLocaleString('en-US', {
+                month: 'short',
+                day: 'numeric',
+                year: 'numeric',
+                hour: '2-digit',
+                minute: '2-digit',
+            });
+            const statusEmoji = { applied: '📤', interview: '🎯', offer: '🎉', rejected: '❌', withdrawn: '🚫' }[app.status] ?? '';
+            console.log(` ${pad(date, colDate)}  ${pad(`${statusEmoji} ${app.status}`, colStatus)}  ${pad(app.title, colTitle)}  ${pad(app.company, colCompany)}  ${pad(app.mode, colMode)}`);
+        }
+        const today = new Date().toISOString().slice(0, 10);
+        const todayCount = allApps.filter(a => a.appliedAt.startsWith(today)).length;
+        console.log(`\n  Today: ${todayCount} application(s)\n`);
+        process.exit(0);
+    }
+    catch (error) {
         console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
         process.exit(1);
     }
@@ -1451,7 +2239,7 @@ program
     .option('--format <fmt>', 'Image format: png (default) or jpeg', 'png')
     .option('--quality <n>', 'JPEG quality 1-100 (ignored for PNG)', parseInt)
     .option('-w, --wait <ms>', 'Wait time after page load (ms)', parseInt)
-    .option('-t, --timeout <ms>', 'Request timeout (ms)', parseInt, 30000)
+    .option('-t, --timeout <ms>', 'Request timeout (ms)', (v) => parseInt(v, 10), 30000)
     .option('--stealth', 'Use stealth mode to bypass bot detection')
     .option('--action <actions...>', 'Page actions before screenshot (e.g., "click:.btn" "wait:2000")')
     .option('-o, --output <path>', 'Output file path (default: screenshot.png)')
@@ -1558,11 +2346,376 @@ program
         process.exit(1);
     }
 });
+// ── Top-level Apply command group ──────────────────────────────────────────
+//
+// webpeel apply <url>          — submit a job application
+// webpeel apply init           — interactive profile setup wizard
+// webpeel apply status         — show application stats
+// webpeel apply list           — list tracked applications (with filters)
+// webpeel apply rate           — show rate-governor status
+const applyCmd = program
+    .command('apply')
+    .description('Auto-apply pipeline: submit applications, track history, manage rate limits');
+// apply <url>  — auto-apply to a job posting
+applyCmd
+    .command('submit <url>')
+    .description('Auto-apply to a job posting')
+    .alias('s')
+    .option('--profile-path <path>', 'Path to apply profile JSON', `${process.env.HOME ?? '~'}/.webpeel/profile.json`)
+    .option('--browser-profile <path>', 'Path to persistent browser data dir', `${process.env.HOME ?? '~'}/.webpeel/browser-profile`)
+    .option('--headed', 'Run browser visibly (default for apply)')
+    .option('--headless', 'Run browser invisibly')
+    .option('--confirm', 'Pause for confirmation before submit (default: true)')
+    .option('--no-confirm', 'Skip confirmation, auto-submit')
+    .option('--dry-run', 'Go through flow but do not submit')
+    .option('--generate-cover', 'Generate tailored cover letter (needs OPENAI_API_KEY)')
+    .option('--timeout <ms>', 'Timeout in ms (default: 300000)', '300000')
+    .option('--json', 'Output result as JSON')
+    .option('--silent', 'Silent mode')
+    .action(async (url, options) => {
+    const isSilent = options.silent;
+    const isJson = options.json;
+    // Load profile
+    const profilePath = options.profilePath;
+    let profile;
+    try {
+        const raw = readFileSync(profilePath, 'utf-8');
+        profile = JSON.parse(raw);
+    }
+    catch {
+        const msg = `Could not load profile from ${profilePath}. Run "webpeel apply init" to create one.`;
+        if (isJson) {
+            await writeStdout(JSON.stringify({ error: msg }) + '\n');
+        }
+        else {
+            console.error(`Error: ${msg}`);
+        }
+        process.exit(1);
+    }
+    const spinner = isSilent ? null : ora('Applying...').start();
+    try {
+        const { applyToJob } = await import('./core/apply.js');
+        const result = await applyToJob({
+            url,
+            profile,
+            // Use sessionDir for persistent session storage (renamed from browserProfile)
+            sessionDir: options.browserProfile,
+            // Map dryRun flag → mode: 'dry-run'
+            mode: (options.dryRun ? 'dry-run' : (options.noConfirm ? 'auto' : 'review')),
+            timeout: parseInt(options.timeout, 10) || 300_000,
+        });
+        if (spinner)
+            spinner.stop();
+        // Normalize result to a consistent output shape
+        const success = result.submitted && !result.error;
+        const jobTitle = result.job?.title ?? '';
+        const jobCompany = result.job?.company ?? '';
+        if (isJson) {
+            await writeStdout(JSON.stringify(result, null, 2) + '\n');
+            process.exit(success ? 0 : 1);
+        }
+        const icon = success ? '✅' : '❌';
+        console.log(`\n${icon} ${success ? 'Application submitted!' : `Failed: ${result.error ?? 'Unknown error'}`}`);
+        if (jobTitle)
+            console.log(`   ${jobTitle}${jobCompany ? ` @ ${jobCompany}` : ''}`);
+        if (options.dryRun)
+            console.log('   (Dry run — not submitted)');
+        console.log(`   Time: ${(result.elapsed / 1000).toFixed(1)}s\n`);
+        process.exit(success ? 0 : 1);
+    }
+    catch (error) {
+        if (spinner)
+            spinner.fail('Application failed');
+        const msg = error instanceof Error ? error.message : 'Unknown error';
+        if (isJson) {
+            await writeStdout(JSON.stringify({ error: msg }) + '\n');
+        }
+        else {
+            console.error(`Error: ${msg}`);
+        }
+        process.exit(1);
+    }
+});
+// apply init  — interactive profile setup
+applyCmd
+    .command('init')
+    .description('Interactive profile setup — creates ~/.webpeel/profile.json')
+    .action(async () => {
+    const { createInterface } = await import('readline');
+    const rl = createInterface({ input: process.stdin, output: process.stdout });
+    const ask = (q) => new Promise((resolve) => rl.question(q, (ans) => resolve(ans.trim())));
+    console.log('\n🤖 WebPeel Apply Setup — Create your applicant profile\n');
+    console.log('This creates ~/.webpeel/profile.json used by "webpeel apply submit".\n');
+    try {
+        const name = await ask('Full name: ');
+        const email = await ask('Email address: ');
+        const phone = await ask('Phone number (optional): ');
+        const resumePath = await ask('Path to resume PDF (e.g. /Users/you/resume.pdf): ');
+        const currentTitle = await ask('Current/most recent job title: ');
+        const yearsExp = await ask('Years of experience: ');
+        const skills = await ask('Skills (comma-separated, e.g. TypeScript, React, Node.js): ');
+        const education = await ask('Education (e.g. B.S. Computer Science, MIT): ');
+        const location = await ask('City, State (e.g. San Francisco, CA): ');
+        const workAuth = await ask('Work authorization (e.g. US Citizen, Permanent Resident, H-1B, Need Sponsorship): ');
+        const linkedinUrl = await ask('LinkedIn URL (optional): ');
+        const websiteUrl = await ask('Portfolio/website URL (optional): ');
+        const desiredSalary = await ask('Desired salary (optional, e.g. $150,000): ');
+        rl.close();
+        const { mkdirSync: mk, writeFileSync: wf } = await import('fs');
+        const { join: j } = await import('path');
+        const { homedir: hd } = await import('os');
+        const webpeelDir = j(hd(), '.webpeel');
+        mk(webpeelDir, { recursive: true });
+        const profile = {
+            name,
+            email,
+            ...(phone ? { phone } : {}),
+            resumePath,
+            currentTitle,
+            yearsExperience: parseInt(yearsExp, 10) || 0,
+            skills: skills.split(',').map((s) => s.trim()).filter(Boolean),
+            education,
+            location,
+            workAuthorization: workAuth,
+            ...(linkedinUrl ? { linkedinUrl } : {}),
+            ...(websiteUrl ? { websiteUrl } : {}),
+            ...(desiredSalary ? { desiredSalary } : {}),
+        };
+        const profilePath = j(webpeelDir, 'profile.json');
+        wf(profilePath, JSON.stringify(profile, null, 2), 'utf-8');
+        console.log(`\n✅ Profile saved to: ${profilePath}`);
+        console.log('\nNext steps:');
+        console.log('  • Apply to a job:  webpeel apply submit <url>');
+        console.log('  • Dry run first:   webpeel apply submit <url> --dry-run');
+        console.log('  • View stats:      webpeel apply status\n');
+    }
+    catch (error) {
+        rl.close();
+        console.error(`\nError: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        process.exit(1);
+    }
+});
+// apply status  — application stats summary
+applyCmd
+    .command('status')
+    .description('Show application stats')
+    .option('--json', 'Output as JSON')
+    .action(async (options) => {
+    try {
+        const { ApplicationTracker } = await import('./core/application-tracker.js');
+        const tracker = new ApplicationTracker();
+        const stats = tracker.stats();
+        if (options.json) {
+            await writeStdout(JSON.stringify(stats, null, 2) + '\n');
+            process.exit(0);
+        }
+        console.log('\n📊 Application Stats\n');
+        console.log(`  Total:     ${stats.total}`);
+        console.log(`  Today:     ${stats.today}`);
+        console.log(`  This week: ${stats.thisWeek}`);
+        if (Object.keys(stats.byPlatform).length > 0) {
+            console.log('\n  By Platform:');
+            for (const [platform, count] of Object.entries(stats.byPlatform)) {
+                console.log(`    ${platform.padEnd(12)} ${count}`);
+            }
+        }
+        if (Object.keys(stats.byStatus).length > 0) {
+            console.log('\n  By Status:');
+            for (const [status, count] of Object.entries(stats.byStatus)) {
+                console.log(`    ${status.padEnd(12)} ${count}`);
+            }
+        }
+        console.log('');
+        process.exit(0);
+    }
+    catch (error) {
+        console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        process.exit(1);
+    }
+});
+// apply list  — list applications with optional filters
+applyCmd
+    .command('list')
+    .description('List tracked applications')
+    .option('--platform <platform>', 'Filter by platform (e.g. linkedin, upwork)')
+    .option('--status <status>', 'Filter by status (applied, interview, rejected, offer, ...)')
+    .option('--since <date>', 'Filter to applications on or after this date (YYYY-MM-DD)')
+    .option('--json', 'Output as JSON')
+    .option('--limit <n>', 'Max records to show (default: 50)', '50')
+    .action(async (options) => {
+    try {
+        const { ApplicationTracker } = await import('./core/application-tracker.js');
+        const tracker = new ApplicationTracker();
+        const limit = parseInt(options.limit, 10) || 50;
+        const records = tracker.list({
+            platform: options.platform,
+            status: options.status,
+            since: options.since,
+        }).slice(0, limit);
+        if (options.json) {
+            await writeStdout(JSON.stringify(records, null, 2) + '\n');
+            process.exit(0);
+        }
+        if (records.length === 0) {
+            console.log('\nNo applications found.\n');
+            process.exit(0);
+        }
+        console.log(`\n📋 Applications (${records.length})\n`);
+        const colDate = 12;
+        const colStatus = 10;
+        const colTitle = 35;
+        const colCompany = 20;
+        const pad = (s, w) => s.length > w ? s.slice(0, w - 1) + '…' : s.padEnd(w);
+        console.log(` ${'Date'.padEnd(colDate)}  ${'Status'.padEnd(colStatus)}  ${'Title'.padEnd(colTitle)}  ${'Company'.padEnd(colCompany)}`);
+        console.log(` ${'-'.repeat(colDate)}  ${'-'.repeat(colStatus)}  ${'-'.repeat(colTitle)}  ${'-'.repeat(colCompany)}`);
+        for (const r of records) {
+            const dateStr = r.appliedAt.slice(0, 10);
+            console.log(` ${pad(dateStr, colDate)}  ${pad(r.status, colStatus)}  ${pad(r.title, colTitle)}  ${pad(r.company, colCompany)}`);
+        }
+        console.log('');
+        process.exit(0);
+    }
+    catch (error) {
+        console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        process.exit(1);
+    }
+});
+// apply rate  — rate governor status
+applyCmd
+    .command('rate')
+    .description('Show rate governor status (daily limits, cooldown, next allowed time)')
+    .option('--json', 'Output as JSON')
+    .option('--reset-cooldown', 'Clear any active cooldown (manual override)')
+    .action(async (options) => {
+    try {
+        const { RateGovernor, formatDuration } = await import('./core/rate-governor.js');
+        const governor = new RateGovernor();
+        if (options.resetCooldown) {
+            governor.resetCooldown();
+            console.log('✅ Cooldown cleared.');
+            process.exit(0);
+        }
+        const state = governor.getState();
+        const config = governor.getConfig();
+        const check = governor.canApply();
+        if (options.json) {
+            await writeStdout(JSON.stringify({
+                state,
+                config,
+                canApply: check.allowed,
+                reason: check.reason,
+                waitMs: check.waitMs,
+                nextDelayMs: governor.getNextDelay(),
+            }, null, 2) + '\n');
+            process.exit(0);
+        }
+        console.log('\n⏱  Rate Governor Status\n');
+        console.log(`  Today's applications: ${state.todayCount} / ${config.maxPerDay}`);
+        console.log(`  Total applications:   ${state.totalApplications}`);
+        console.log(`  Can apply now:        ${check.allowed ? '✅ Yes' : '❌ No'}`);
+        if (!check.allowed && check.reason) {
+            console.log(`  Reason:               ${check.reason}`);
+        }
+        if (!check.allowed && check.waitMs) {
+            console.log(`  Wait time:            ${formatDuration(check.waitMs)}`);
+        }
+        if (state.cooldownUntil > 0) {
+            const remaining = state.cooldownUntil - Date.now();
+            console.log(`  Cooldown:             Active (${formatDuration(Math.max(0, remaining))} remaining)`);
+        }
+        console.log(`  Min delay:            ${formatDuration(config.minDelayMs)}`);
+        console.log(`  Max delay:            ${formatDuration(config.maxDelayMs)}`);
+        console.log(`  Active hours:         ${config.activeHours[0]}:00 – ${config.activeHours[1]}:00`);
+        console.log(`  Weekdays only:        ${config.weekdaysOnly ? 'Yes' : 'No'}`);
+        console.log('');
+        process.exit(0);
+    }
+    catch (error) {
+        console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        process.exit(1);
+    }
+});
 program.parse();
 // ============================================================
+// Time formatting helper
+// ============================================================
+/**
+ * Format a past Date relative to now (e.g. "2h ago", "5m ago").
+ */
+function formatRelativeTime(past) {
+    const diffMs = Date.now() - past.getTime();
+    const diffSec = Math.round(diffMs / 1000);
+    if (diffSec < 60)
+        return `${diffSec}s ago`;
+    const diffMin = Math.round(diffSec / 60);
+    if (diffMin < 60)
+        return `${diffMin}m ago`;
+    const diffHr = Math.round(diffMin / 60);
+    if (diffHr < 24)
+        return `${diffHr}h ago`;
+    const diffDay = Math.round(diffHr / 24);
+    return `${diffDay}d ago`;
+}
+// ============================================================
+// Error classification for JSON error output (#6)
+// ============================================================
+function classifyErrorCode(error) {
+    if (!(error instanceof Error))
+        return 'FETCH_FAILED';
+    // Check for our custom _code first (set in pre-fetch validation)
+    if (error._code)
+        return error._code;
+    const msg = error.message.toLowerCase();
+    const name = error.name || '';
+    if (name === 'TimeoutError' || msg.includes('timeout') || msg.includes('timed out')) {
+        return 'TIMEOUT';
+    }
+    if (name === 'BlockedError' || msg.includes('blocked') || msg.includes('403') || msg.includes('cloudflare')) {
+        return 'BLOCKED';
+    }
+    if (msg.includes('enotfound') || msg.includes('getaddrinfo') || msg.includes('dns resolution failed') || msg.includes('not found')) {
+        return 'DNS_FAILED';
+    }
+    if (msg.includes('invalid url') || msg.includes('invalid hostname') || msg.includes('only http')) {
+        return 'INVALID_URL';
+    }
+    return 'FETCH_FAILED';
+}
+/**
+ * Build a unified PeelEnvelope from a PeelResult.
+ *
+ * All existing PeelResult fields are spread first (backward compatibility),
+ * then canonical envelope fields override/extend them.
+ */
+function buildEnvelope(result, extra) {
+    const envelope = {
+        // Spread all PeelResult fields for backward compatibility
+        ...result,
+        // Required envelope fields (override PeelResult where they overlap)
+        url: result.url,
+        status: 200,
+        content: result.content,
+        metadata: {
+            title: result.title,
+            ...result.metadata,
+        },
+        tokens: result.tokens,
+        cached: extra.cached ?? false,
+        elapsed: result.elapsed,
+    };
+    // Optional envelope fields — only include when meaningful
+    if (extra.structured !== undefined)
+        envelope.structured = extra.structured;
+    if (extra.truncated)
+        envelope.truncated = true;
+    if (extra.totalAvailable !== undefined)
+        envelope.totalAvailable = extra.totalAvailable;
+    return envelope;
+}
+// ============================================================
 // Shared output helper
 // ============================================================
-async function outputResult(result, options) {
+async function outputResult(result, options, extra = {}) {
     // --links: output only links
     if (options.links) {
         if (options.json) {
@@ -1603,6 +2756,7 @@ async function outputResult(result, options) {
             method: result.method,
             elapsed: result.elapsed,
             tokens: result.tokens,
+            cached: extra.cached ?? false,
             ...result.metadata,
         };
         if (options.json) {
@@ -1624,12 +2778,14 @@ async function outputResult(result, options) {
             console.log(`Method:      ${meta.method}`);
             console.log(`Elapsed:     ${meta.elapsed}ms`);
             console.log(`Tokens:      ${meta.tokens}`);
+            console.log(`Cached:      ${meta.cached}`);
         }
         return;
     }
     // Default: full output
     if (options.json) {
-        await writeStdout(JSON.stringify(result, null, 2) + '\n');
+        const envelope = buildEnvelope(result, extra);
+        await writeStdout(JSON.stringify(envelope, null, 2) + '\n');
     }
     else {
         await writeStdout(result.content + '\n');
@@ -1645,6 +2801,64 @@ function writeStdout(data) {
         });
     });
 }
+/**
+ * Convert an array of listing items to CSV.
+ */
+function formatListingsCsv(items) {
+    if (items.length === 0)
+        return '';
+    // Collect all keys
+    const keySet = new Set();
+    for (const item of items) {
+        for (const key of Object.keys(item)) {
+            if (item[key] !== undefined)
+                keySet.add(key);
+        }
+    }
+    const keys = Array.from(keySet);
+    const escapeCsv = (s) => {
+        if (s === undefined || s === null)
+            return '""';
+        const str = String(s);
+        if (str.includes('"') || str.includes(',') || str.includes('\n') || str.includes('\r')) {
+            return '"' + str.replace(/"/g, '""') + '"';
+        }
+        return '"' + str + '"';
+    };
+    const lines = [keys.join(',')];
+    for (const item of items) {
+        lines.push(keys.map(k => escapeCsv(item[k])).join(','));
+    }
+    return lines.join('\n') + '\n';
+}
+/**
+ * Normalise the result of --extract (which may be a flat object or contain
+ * arrays) into an array of row objects suitable for CSV / table rendering.
+ */
+function normaliseExtractedToRows(extracted) {
+    // If every value is an array of the same length, zip them into rows
+    const values = Object.values(extracted);
+    const allArrays = values.length > 0 && values.every(v => Array.isArray(v));
+    if (allArrays) {
+        const length = values[0].length;
+        const rows = [];
+        for (let i = 0; i < length; i++) {
+            const row = {};
+            for (const key of Object.keys(extracted)) {
+                const val = extracted[key][i];
+                row[key] = val != null ? String(val) : undefined;
+            }
+            rows.push(row);
+        }
+        return rows;
+    }
+    // Otherwise treat as a single row
+    const row = {};
+    for (const [k, v] of Object.entries(extracted)) {
+        row[k] = v != null ? String(v) : undefined;
+    }
+    return [row];
+}
 // Helper function to extract colors from content
 function extractColors(content) {
     const colors = [];