webpeel 0.20.4 → 0.20.5

package/dist/cli/commands/fetch.js

@@ -13,6 +13,14 @@ import { parseActions, formatError, fetchViaApi, outputResult, writeStdout, buil
 // ─── runFetch ─────────────────────────────────────────────────────────────────
 // Main fetch handler — shared with the `pipe` and `ask` subcommands
 export async function runFetch(url, options) {
+    // --content-only: override all output flags — we just want raw content
+    if (options.contentOnly) {
+        options.silent = true;
+        // Disable json/text/html — we output content directly
+        options.json = false;
+        options.html = false;
+        options.text = false;
+    }
     // Handle --format flag: maps to existing boolean flags
     if (options.format) {
         const fmt = options.format.toLowerCase();
@@ -30,9 +38,10 @@ export async function runFetch(url, options) {
     }
     // Smart defaults: when piped (not a TTY), default to silent JSON + budget
     // BUT respect explicit --format flag (user chose the output format)
+    // AND respect --content-only (raw content output, no JSON wrapper)
     const isPiped = !process.stdout.isTTY;
     const hasExplicitFormat = options.format && ['text', 'html', 'markdown', 'md'].includes(options.format.toLowerCase());
-    if (isPiped && !options.html && !options.text && !hasExplicitFormat) {
+    if (isPiped && !options.html && !options.text && !hasExplicitFormat && !options.contentOnly) {
         if (!options.json)
             options.json = true;
         if (!options.silent)
@@ -284,11 +293,38 @@ export async function runFetch(url, options) {
                     cachedResult.extracted = extractedCached;
                 }
             }
-            await outputResult(cachedResult, options, { cached: true });
+            if (options.contentOnly) {
+                await writeStdout(cachedResult.content + '\n');
+            }
+            else {
+                await outputResult(cachedResult, options, { cached: true });
+            }
             process.exit(0);
         }
     }
-    const spinner = options.silent ? null : ora('Fetching...').start();
+    // --progress: show escalation steps on stderr (overrides spinner)
+    let progressInterval;
+    const progressStart = Date.now();
+    if (options.progress) {
+        process.stderr.write(`[simple] Fetching ${url}...\n`);
+        // Show escalation hints based on elapsed time (best-effort approximations)
+        const progressSteps = [
+            { afterMs: 2500, message: '[simple] Waiting for response...' },
+            { afterMs: 6000, message: '[browser] Simple too slow — escalating to browser render...' },
+            { afterMs: 12000, message: '[browser] Rendering with Chromium...' },
+            { afterMs: 20000, message: '[stealth] Escalating to stealth mode...' },
+        ];
+        let stepIdx = 0;
+        progressInterval = setInterval(() => {
+            const elapsed = Date.now() - progressStart;
+            while (stepIdx < progressSteps.length && elapsed >= progressSteps[stepIdx].afterMs) {
+                process.stderr.write(`${progressSteps[stepIdx].message}\n`);
+                stepIdx++;
+            }
+        }, 500);
+    }
+    // Suppress spinner when --progress is active (progress lines replace it)
+    const spinner = (options.silent || options.progress) ? null : ora('Fetching...').start();
     try {
         // Validate options
         if (options.wait && (options.wait < 0 || options.wait > 60000)) {
@@ -528,7 +564,22 @@ export async function runFetch(url, options) {
         if (resolvedProfileName) {
             touchProfile(resolvedProfileName);
         }
-        if (spinner) {
+        // Stop progress interval and show final result
+        if (progressInterval) {
+            clearInterval(progressInterval);
+            progressInterval = undefined;
+        }
+        if (options.progress) {
+            const method = result.method || 'simple';
+            const elapsedSec = ((result.elapsed || (Date.now() - progressStart)) / 1000).toFixed(1);
+            const tokenCount = (result.tokens || 0).toLocaleString();
+            // Show escalation arrow if browser/stealth was needed
+            if (method !== 'simple') {
+                process.stderr.write(`[simple] → [${method}] escalated\n`);
+            }
+            process.stderr.write(`[${method}] Done — ${tokenCount} tokens in ${elapsedSec}s\n`);
+        }
+        else if (spinner) {
             const domainTag = result.domainData
                 ? ` [${result.domainData.domain}:${result.domainData.type}]`
                 : '';
@@ -866,11 +917,27 @@ export async function runFetch(url, options) {
                     result.extracted = extracted;
                 }
             }
-            // Output results (default path)
-            await outputResult(result, options, {
-                cached: false,
-                truncated: contentTruncated || undefined,
-            });
+            // --content-only: output raw content only, no wrapper
+            if (options.contentOnly) {
+                await writeStdout(result.content + '\n');
+            }
+            else {
+                // Output results (default path)
+                await outputResult(result, options, {
+                    cached: false,
+                    truncated: contentTruncated || undefined,
+                });
+                // Token savings display (our unique selling point)
+                if (!options.json && !options.silent && result.tokenSavingsPercent) {
+                    const savings = result.tokenSavingsPercent;
+                    const raw = result.rawTokenEstimate;
+                    const optimized = result.tokens || 0;
+                    if (savings > 0) {
+                        const rawStr = raw ? `${raw.toLocaleString()}→${optimized.toLocaleString()} tokens` : `${optimized.toLocaleString()} tokens`;
+                        process.stderr.write(`\x1b[32m💰 Token savings: ${savings}% smaller than raw HTML (${rawStr})\x1b[0m\n`);
+                    }
+                }
+            }
         }
         // Clean up and exit
         await cleanup();
@@ -976,6 +1043,8 @@ export function registerFetchCommands(program) {
         .option('--wait-selector <css>', 'Wait for CSS selector before extracting (auto-enables --render)')
         .option('--block-resources <types>', 'Block resource types, comma-separated: image,stylesheet,font,media,script (auto-enables --render)')
         .option('--format <type>', 'Output format: markdown (default), text, html, json')
+        .option('--content-only', 'Output only the raw content field (no metadata, no JSON wrapper) — ideal for piping to LLMs')
+        .option('--progress', 'Show engine escalation steps (simple → browser → stealth) with timing')
         .action(async (url, options) => {
         await runFetch(url, options);
     });

package/dist/cli/commands/search.js

@@ -23,6 +23,7 @@ export function registerSearchCommands(program) {
         .option('--budget <n>', 'Token budget for site-search result content', parseInt)
         .option('-s, --silent', 'Silent mode')
         .option('--proxy <url>', 'Proxy URL for requests (http://host:port, socks5://user:pass@host:port)')
+        .option('--fetch', 'Also fetch and include content from each result URL')
         .option('--agent', 'Agent mode: sets --json, --silent, and --budget 4000 (override with --budget N)')
         .action(async (query, options) => {
         // --agent sets sensible defaults for AI agents; explicit flags override
@@ -178,9 +179,61 @@ export function registerSearchCommands(program) {
             const searchData = await searchRes.json();
             // API returns { success: true, data: { web: [...] } } or { results: [...] }
             let results = searchData.data?.web || searchData.data?.results || searchData.results || [];
+            // Client-side ad filtering: remove DuckDuckGo ads that slip through the server
+            results = results.filter(r => {
+                // Filter DDG-internal URLs
+                try {
+                    const parsed = new URL(r.url);
+                    if (parsed.hostname === 'duckduckgo.com')
+                        return false;
+                    if (parsed.searchParams.has('ad_domain') ||
+                        parsed.searchParams.has('ad_provider') ||
+                        parsed.searchParams.has('ad_type'))
+                        return false;
+                }
+                catch {
+                    return false;
+                }
+                // Filter ad snippets
+                if (r.snippet && (r.snippet.includes('Ad ·') ||
+                    r.snippet.includes('Ad Viewing ads is privacy protected by DuckDuckGo') ||
+                    r.snippet.toLowerCase().startsWith('ad ·')))
+                    return false;
+                return true;
+            });
             if (spinner) {
                 spinner.succeed(`Found ${results.length} results`);
             }
+            // --fetch: fetch content from each result
+            if (options.fetch && results.length > 0) {
+                const fetchCfg = loadConfig();
+                const fetchApiKey = fetchCfg.apiKey || process.env.WEBPEEL_API_KEY;
+                const fetchApiUrl = process.env.WEBPEEL_API_URL || 'https://api.webpeel.dev';
+                if (fetchApiKey) {
+                    const fetchSpinner = isSilent ? null : ora(`Fetching content from ${results.length} results...`).start();
+                    await Promise.all(results.map(async (result) => {
+                        try {
+                            const fetchParams = new URLSearchParams({ url: result.url });
+                            if (options.budget)
+                                fetchParams.set('budget', String(options.budget || 2000));
+                            const fetchRes = await fetch(`${fetchApiUrl}/v1/fetch?${fetchParams}`, {
+                                headers: { Authorization: `Bearer ${fetchApiKey}` },
+                                signal: AbortSignal.timeout(20000),
+                            });
+                            if (fetchRes.ok) {
+                                const fetchData = await fetchRes.json();
+                                result.content = fetchData.content || fetchData.data?.content || '';
+                            }
+                        }
+                        catch { /* skip on error */ }
+                    }));
+                    if (fetchSpinner)
+                        fetchSpinner.succeed('Content fetched');
+                }
+                else if (!isSilent) {
+                    console.error('Warning: --fetch requires API key (run: webpeel auth <key>)');
+                }
+            }
             // Show usage footer for free/anonymous users
             if (usageCheck.usageInfo && !isSilent) {
                 showUsageFooter(usageCheck.usageInfo, usageCheck.isAnonymous || false, false);
@@ -196,10 +249,24 @@ export function registerSearchCommands(program) {
                 await writeStdout(jsonStr + '\n');
             }
             else {
-                for (const result of results) {
-                    console.log(`\n${result.title}`);
-                    console.log(result.url);
-                    console.log(result.snippet);
+                // Human-readable numbered results
+                if (results.length === 0) {
+                    await writeStdout('No results found.\n');
+                }
+                else {
+                    await writeStdout(`\n`);
+                    for (const [i, result] of results.entries()) {
+                        await writeStdout(`${i + 1}. ${result.title}\n`);
+                        await writeStdout(`   ${result.url}\n`);
+                        if (result.snippet) {
+                            await writeStdout(`   ${result.snippet}\n`);
+                        }
+                        if (result.content) {
+                            const preview = result.content.slice(0, 500);
+                            await writeStdout(`\n   --- Content ---\n${preview}${result.content.length > 500 ? '\n   [...]' : ''}\n`);
+                        }
+                        await writeStdout('\n');
+                    }
                 }
             }
             process.exit(0);

package/dist/cli/utils.js

@@ -508,13 +508,11 @@ export async function outputResult(result, options, extra = {}) {
     // Default: full output
     if (options.json) {
         // Build clean JSON output with guaranteed top-level fields
+        // Note: elapsed/method/tokens are placed at the END so `tail -3` shows perf metrics
         const output = {
             url: result.url,
             title: result.metadata?.title || result.title || null,
-            tokens: result.tokens || 0,
             fetchedAt: new Date().toISOString(),
-            method: result.method || 'simple',
-            elapsed: result.elapsed,
             content: result.content,
         };
         // Add optional fields only if present (filter out undefined/null values from metadata)
@@ -529,6 +527,10 @@ export async function outputResult(result, options, extra = {}) {
         }
         if (result.links?.length)
             output.links = result.links;
+        if (result.tokenSavingsPercent !== undefined)
+            output.tokenSavingsPercent = result.tokenSavingsPercent;
+        if (result.rawTokenEstimate !== undefined)
+            output.rawTokenEstimate = result.rawTokenEstimate;
         if (result.images?.length)
             output.images = result.images;
         if (result.structured)
@@ -562,6 +564,10 @@ export async function outputResult(result, options, extra = {}) {
         if (extra.totalAvailable !== undefined)
             output.totalAvailable = extra.totalAvailable;
         output._meta = { version: cliVersion, method: result.method || 'simple', timing: result.timing, serverMarkdown: result.serverMarkdown || false };
+        // Perf metrics at the end — `tail -3` shows: elapsed | method | tokens
+        output.elapsed = result.elapsed;
+        output.method = result.method || 'simple';
+        output.tokens = result.tokens || 0;
         await writeStdout(JSON.stringify(output, null, 2) + '\n');
     }
     else {
@@ -586,10 +592,11 @@ export async function outputResult(result, options, extra = {}) {
         }
         // Stream content immediately to stdout — consumer gets it without waiting
         await writeStdout(result.content + '\n');
-        // Append timing summary to stderr so it doesn't pollute piped content
-        if (!options.silent) {
+        // Append timing summary to stderr (always — doesn't pollute stdout pipe)
+        {
             const totalMs = result.timing?.total ?? result.elapsed;
-            process.stderr.write(`\n--- ${result.tokens} tokens · ${totalMs}ms ---\n`);
+            const method = result.method || 'simple';
+            process.stderr.write(`\n--- ${totalMs}ms | ${method} | ${result.tokens} tokens ---\n`);
         }
     }
 }

package/dist/core/search-provider.d.ts

@@ -83,7 +83,7 @@ export declare const providerStats: ProviderStatsTracker;
 export declare class StealthSearchProvider implements SearchProvider {
     readonly id: SearchProviderId;
     readonly requiresApiKey = false;
-    /** Validate and normalize a URL; returns null if invalid/non-http */
+    /** Validate and normalize a URL; returns null if invalid/non-http or a DDG ad URL */
     private validateUrl;
     /**
      * Scrape DuckDuckGo HTML endpoint with stealth browser.
@@ -145,7 +145,7 @@ export declare class GoogleSearchProvider implements SearchProvider {
      *                      m[n]=past n months, y[n]=past n years.
      */
     private mapFreshnessToDateRestrict;
-    /** Validate URL; returns null if invalid/non-http */
+    /** Validate URL; returns null if invalid/non-http or a DDG ad URL */
     private validateUrl;
     /**
      * Stealth browser scrape of google.com/search.

package/dist/core/search-provider.js

@@ -84,6 +84,30 @@ function decodeDdgUrl(rawUrl) {
         return rawUrl;
     }
 }
+/** Returns true if a URL looks like a DuckDuckGo ad or tracking link */
+function isDdgAdUrl(url) {
+    try {
+        const parsed = new URL(url);
+        // DDG-internal ad redirect paths
+        if (parsed.hostname === 'duckduckgo.com')
+            return true;
+        // URLs with known ad tracking query params
+        if (parsed.searchParams.has('ad_domain') ||
+            parsed.searchParams.has('ad_provider') ||
+            parsed.searchParams.has('ad_type'))
+            return true;
+        return false;
+    }
+    catch {
+        return false;
+    }
+}
+/** Returns true if a snippet is a DuckDuckGo ad snippet */
+function isDdgAdSnippet(snippet) {
+    return snippet.includes('Ad ·') ||
+        snippet.includes('Ad Viewing ads is privacy protected by DuckDuckGo') ||
+        snippet.toLowerCase().startsWith('ad ·');
+}
 class ProviderStatsTracker {
     history = new Map();
     windowSize;
@@ -182,14 +206,19 @@ function normalizeUrlForDedupe(rawUrl) {
 export class StealthSearchProvider {
     id = 'stealth';
     requiresApiKey = false;
-    /** Validate and normalize a URL; returns null if invalid/non-http */
+    /** Validate and normalize a URL; returns null if invalid/non-http or a DDG ad URL */
     validateUrl(rawUrl) {
         try {
             const parsed = new URL(rawUrl);
             if (!['http:', 'https:'].includes(parsed.protocol))
                 return null;
-            // Filter DuckDuckGo ad redirect URLs (e.g. duckduckgo.com/y.js?ad_domain=...)
-            if (parsed.hostname === 'duckduckgo.com' && parsed.pathname === '/y.js')
+            // Filter all DuckDuckGo URLs (internal links, ad redirects, etc.)
+            if (parsed.hostname === 'duckduckgo.com')
+                return null;
+            // Filter URLs with ad tracking query params
+            if (parsed.searchParams.has('ad_domain') ||
+                parsed.searchParams.has('ad_provider') ||
+                parsed.searchParams.has('ad_type'))
                 return null;
             return parsed.href;
         }
@@ -236,10 +265,16 @@ export class StealthSearchProvider {
                 const snippet = cleanText(snippetRaw, { maxLen: 500, stripEllipsisPadding: true });
                 if (!title || !rawUrl)
                     return;
+                // Filter ad snippets
+                if (isDdgAdSnippet(snippet))
+                    return;
                 // Extract real URL from DDG redirect param
                 const finalUrl = decodeDdgUrl(rawUrl);
                 if (!finalUrl)
                     return; // filtered out (DDG internal link)
+                // Filter ad URLs
+                if (isDdgAdUrl(finalUrl))
+                    return;
                 const validated = this.validateUrl(finalUrl);
                 if (!validated)
                     return;
@@ -532,10 +567,16 @@ export class DuckDuckGoProvider {
             let snippet = cleanText(snippetRaw, { maxLen: 500, stripEllipsisPadding: true });
             if (!title || !rawUrl)
                 return;
+            // Filter ad snippets (DuckDuckGo injects ad labels into snippets)
+            if (isDdgAdSnippet(snippet))
+                return;
             // Extract actual URL from DuckDuckGo redirect; filter DDG internal/ad URLs
             const decoded = decodeDdgUrl(rawUrl);
             if (!decoded)
                 return; // filtered out (DDG internal link or ad redirect)
+            // Filter ad URLs
+            if (isDdgAdUrl(decoded))
+                return;
             // SECURITY: Validate and sanitize results — only allow HTTP/HTTPS URLs
             let url;
             try {
@@ -813,14 +854,19 @@ export class GoogleSearchProvider {
         };
         return map[tbs];
     }
-    /** Validate URL; returns null if invalid/non-http */
+    /** Validate URL; returns null if invalid/non-http or a DDG ad URL */
     validateUrl(rawUrl) {
         try {
             const parsed = new URL(rawUrl);
             if (!['http:', 'https:'].includes(parsed.protocol))
                 return null;
-            // Filter DuckDuckGo ad redirect URLs (e.g. duckduckgo.com/y.js?ad_domain=...)
-            if (parsed.hostname === 'duckduckgo.com' && parsed.pathname === '/y.js')
+            // Filter all DuckDuckGo URLs (internal links, ad redirects, etc.)
+            if (parsed.hostname === 'duckduckgo.com')
+                return null;
+            // Filter URLs with ad tracking query params
+            if (parsed.searchParams.has('ad_domain') ||
+                parsed.searchParams.has('ad_provider') ||
+                parsed.searchParams.has('ad_type'))
                 return null;
             return parsed.href;
         }

package/dist/core/strategies.js

@@ -597,8 +597,12 @@ export async function smartFetch(url, options = {}) {
                     .then((result) => ({ type: 'simple-success', result }))
                     .catch((error) => ({ type: 'simple-error', error }));
                 if (simpleResult.type === 'simple-success') {
-                    // Check if the content is suspiciously thin or has SPA indicators — escalate to browser if so
-                    if (shouldEscalateForLowContent(simpleResult.result) || hasSpaIndicators(simpleResult.result.html)) {
+                    // Check if the content is suspiciously thin, looks like an SPA shell, or is a shell page
+                    // (looksLikeShellPage catches partial renders with 200-500 visible chars that
+                    // shouldEscalateForLowContent misses — improves consistency on sites like China Daily)
+                    if (shouldEscalateForLowContent(simpleResult.result) ||
+                        hasSpaIndicators(simpleResult.result.html) ||
+                        looksLikeShellPage(simpleResult.result)) {
                         shouldUseBrowser = true;
                     }
                     else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.20.4",
+  "version": "0.20.5",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",