webpeel 0.21.86 → 0.21.87

package/dist/cli/commands/fetch.js

@@ -289,6 +289,7 @@ export async function runFetch(url, options) {
             format: options.html ? 'html' : options.text ? 'text' : options.clean ? 'clean' : 'markdown',
             budget: null, // Budget excluded from cache key — cache stores full content
             readable: options.readable || false,
+            noDomainApi: options.skipDomainApi || false, // Different cache for domain-api bypass
         };
         const cachedResult = getCache(url, cacheOptions);
         if (cachedResult) {
@@ -603,6 +604,7 @@ export async function runFetch(url, options) {
             headers,
             cookies: options.cookie,
             raw: options.raw || false,
+            noDomainApi: options.skipDomainApi || false,
             lite: options.lite || false,
             actions,
             maxTokens: options.maxTokens,
@@ -724,6 +726,16 @@ export async function runFetch(url, options) {
                 ? ` [${result.domainData.domain}:${result.domainData.type}]`
                 : '';
             spinner.succeed(`Fetched in ${result.elapsed}ms using ${result.method} method${domainTag}`);
+            // Smart hints — suggest features the user might not know about
+            if (!options.silent && !options.json && !options.skipDomainApi) {
+                if (result.method === 'domain-api') {
+                    const extractorName = result.domainData?.domain || new URL(url).hostname.replace('www.', '') || 'domain';
+                    console.error(`\x1b[33m💡 Tip: Using our ${extractorName} extractor. Want the raw page instead? Add --skip-domain-api\x1b[0m`);
+                }
+            }
+            if (!options.silent && !options.json && result.tokens && result.tokens < 50 && !options.render) {
+                console.error(`\x1b[33m💡 Tip: Page returned very little content. Try --render for JavaScript-heavy sites or --stealth if blocked.\x1b[0m`);
+            }
         }
         // Show metadata header
         const pageTitle = result.metadata?.title || result.title;
@@ -1176,6 +1188,7 @@ export function registerFetchCommands(program) {
         .option('--images', 'Output image URLs from the page')
         .option('--meta', 'Output only the page metadata (title, description, author, etc.)')
         .option('--raw', 'Return full page without smart content extraction')
+        .option('--skip-domain-api', 'Bypass domain-specific API extractors — force actual page scraping')
         .option('--full', 'Alias for --raw — full page content, no budget')
         .option('--lite', 'Lite mode — minimal processing, maximum speed (skip pruning, budget, metadata)')
         .option('--action <actions...>', 'Page actions before scraping (e.g., "click:.btn" "wait:2000" "scroll:bottom")')

package/dist/cli/utils.js

@@ -35,7 +35,14 @@ export async function checkForUpdates() {
         if (latest && latest !== cliVersion && cliVersion !== '0.0.0') {
             // Skip update notice in silent mode
             if (process.env.WEBPEEL_LOG_LEVEL !== 'silent') {
-                console.error(`\n💡 WebPeel v${latest} available (you have v${cliVersion}). Update: npm i -g webpeel@latest\n`);
+                const msg = `Update available: ${cliVersion} → ${latest}`;
+                const cmd = 'npm i -g webpeel@latest';
+                const width = Math.max(msg.length, cmd.length) + 4;
+                const line = '─'.repeat(width);
+                console.error(`\n\x1b[33m╭${line}╮\x1b[0m`);
+                console.error(`\x1b[33m│\x1b[0m  ${msg.padEnd(width - 2)}  \x1b[33m│\x1b[0m`);
+                console.error(`\x1b[33m│\x1b[0m  Run: \x1b[36m${cmd}\x1b[0m${' '.repeat(width - 6 - cmd.length)}  \x1b[33m│\x1b[0m`);
+                console.error(`\x1b[33m╰${line}╯\x1b[0m\n`);
             }
         }
     }
@@ -208,6 +215,8 @@ export async function fetchViaApi(url, options, apiKey, apiUrl) {
         params.set('budget', String(options.budget));
     if (options.question)
         params.set('question', options.question);
+    if (options.noDomainApi)
+        params.set('noDomainApi', 'true');
     const res = await fetch(`${apiUrl}/v1/fetch?${params}`, {
         headers: { Authorization: `Bearer ${apiKey}` },
         signal: AbortSignal.timeout(60000),

package/dist/core/http-fetch.js

@@ -154,10 +154,12 @@ export function createAbortError() {
  * proxy when proxy credentials are configured (WEBSHARE_PROXY_* env vars).
  */
 export const PROXY_PREFERRED_DOMAINS = [
+    // Social / content
     'reddit.com',
     'old.reddit.com',
     'forbes.com',
     'fortune.com',
+    // Auto / cars
     'cargurus.com',
     'edmunds.com',
     'cars.com',
@@ -165,14 +167,29 @@ export const PROXY_PREFERRED_DOMAINS = [
     'autotrader.com',
     'carfax.com',
     'tesla.com',
+    'motortrend.com',
+    'jdpower.com',
+    // Finance / home
     'nerdwallet.com',
     'bankrate.com',
     'homeadvisor.com',
     'angi.com',
+    // EV / auto news
     'insideevs.com',
     'electrek.co',
-    'motortrend.com',
-    'jdpower.com',
+    // Restaurants / food
+    'yelp.com',
+    // Travel
+    'kayak.com',
+    'booking.com',
+    'expedia.com',
+    'tripadvisor.com',
+    'hotels.com',
+    // Shopping / products
+    'amazon.com',
+    'bestbuy.com',
+    'walmart.com',
+    'target.com',
 ];
 /**
  * Returns true if the URL's domain is on the proxy-preferred blocklist.

package/dist/core/pipeline.js

@@ -341,7 +341,8 @@ export async function fetchContent(ctx) {
     const needsDesignAnalysis = ctx.options.designAnalysis && ctx.render;
     // Try API-based domain extraction first (Reddit, GitHub, HN use APIs, not HTML)
     // This avoids expensive browser fetches that often get blocked
-    if (hasDomainExtractor(ctx.url)) {
+    // Skip if noDomainApi is set — user wants raw page content, not API shortcut
+    if (hasDomainExtractor(ctx.url) && !ctx.options.noDomainApi) {
         try {
             ctx.timer.mark('domainApiFirst');
             const ddResult = await runDomainExtract('', ctx.url);
@@ -1078,7 +1079,7 @@ export async function postProcess(ctx) {
     }
     // Domain-aware structured extraction (Twitter, Reddit, GitHub, HN)
     // Fires when URL matches a known domain. Replaces content with clean markdown.
-    if (hasDomainExtractor(fetchResult.url) && !ctx.domainApiHandled) {
+    if (hasDomainExtractor(fetchResult.url) && !ctx.domainApiHandled && !ctx.options.noDomainApi) {
         try {
             ctx.timer.mark('domainExtract');
             // Try raw HTML first, then fall back to readability-processed content

package/dist/core/schema-templates.js

@@ -57,42 +57,55 @@ export const SCHEMA_TEMPLATES = {
         name: 'Event',
         description: 'Extract event information',
         fields: {
-            name: 'event name or title',
-            date: 'event date and time',
-            location: 'venue or location',
-            description: 'event description',
-            price: 'ticket price or cost',
-            organizer: 'event organizer',
-            url: 'registration or ticket URL',
+            name: 'What is the name of this event?',
+            date: 'When does this event take place?',
+            time: 'What time does this event start?',
+            location: 'Where is this event held?',
+            price: 'How much does this event cost?',
+            description: 'What is this event about?',
+            organizer: 'Who is organizing this event?',
         },
     },
     recipe: {
         name: 'Recipe',
         description: 'Extract recipe information from cooking sites',
         fields: {
-            title: 'recipe name',
-            ingredients: 'list of ingredients with quantities',
-            instructions: 'cooking steps or directions',
-            prepTime: 'preparation time',
-            cookTime: 'cooking time',
-            servings: 'number of servings',
-            calories: 'calories per serving',
-            author: 'recipe author or source',
+            name: 'What is the name of this recipe?',
+            ingredients: 'What ingredients are needed? List all.',
+            steps: 'What are the cooking steps or instructions?',
+            prepTime: 'How long does preparation take?',
+            cookTime: 'How long does cooking take?',
+            servings: 'How many servings does this recipe make?',
+            calories: 'How many calories per serving?',
+            rating: 'What is the recipe rating?',
         },
     },
     job: {
         name: 'Job',
         description: 'Extract job posting information',
         fields: {
-            title: 'job title',
-            company: 'company name',
-            location: 'job location',
-            salary: 'salary range or compensation',
-            description: 'job description',
-            requirements: 'required qualifications or skills',
-            type: 'job type (full-time, part-time, remote)',
-            posted: 'date posted',
-            applyUrl: 'application URL or link',
+            title: 'What is the job title?',
+            company: 'What company is hiring?',
+            location: 'Where is the job located?',
+            salary: 'What is the salary or compensation range?',
+            type: 'Is this full-time, part-time, contract, or remote?',
+            requirements: 'What are the key requirements or qualifications?',
+            description: 'What is the job description?',
+            applyUrl: 'What is the URL or method to apply?',
+        },
+    },
+    business: {
+        name: 'Business',
+        description: 'Extract business/company information',
+        fields: {
+            name: 'What is the business name?',
+            address: 'What is the full address?',
+            phone: 'What is the phone number?',
+            hours: 'What are the business hours?',
+            rating: 'What is the business rating?',
+            reviewCount: 'How many reviews does this business have?',
+            website: 'What is the business website URL?',
+            categories: 'What type of business is this?',
         },
     },
     review: {

package/dist/core/search-provider.d.ts

@@ -19,6 +19,8 @@ export interface WebSearchResult {
     snippet: string;
     /** Relevance score (0–1) based on keyword overlap with query. Added by filterRelevantResults. */
     relevanceScore?: number;
+    /** Thumbnail/image URL from SearXNG results (img_src or thumbnail field). */
+    imageUrl?: string;
 }
 export interface WebSearchOptions {
     /** Number of results (1-10) */

package/dist/core/search-provider.js

@@ -1066,8 +1066,15 @@ export class DuckDuckGoProvider {
                 if (searxResults.length > 0) {
                     providerStats.record('searxng', true);
                     log.debug(`source=searxng returned ${searxResults.length} results`);
-                    const filtered = filterRelevantResults(searxResults, query);
-                    return filtered.length > 0 ? filtered : searxResults;
+                    // Map SearXNG results to WebSearchResult (description → snippet, imageUrl passthrough)
+                    const mapped = searxResults.map(r => ({
+                        title: r.title,
+                        url: r.url,
+                        snippet: r.description ?? '',
+                        imageUrl: r.imageUrl,
+                    }));
+                    const filtered = filterRelevantResults(mapped, query);
+                    return filtered.length > 0 ? filtered : mapped;
                 }
                 providerStats.record('searxng', false);
                 log.debug('SearXNG returned 0 results, falling through to DDG');

package/dist/core/searxng-provider.d.ts

@@ -16,6 +16,7 @@ export interface SearXNGSearchResult {
     description?: string;
     publishedDate?: string;
     score?: number;
+    imageUrl?: string;
 }
 /**
  * Fetches search results from a SearXNG instance.

package/dist/core/searxng-provider.js

@@ -69,6 +69,7 @@ export async function searchViaSearXNG(query, options = {}) {
                 description: r.content ?? undefined,
                 publishedDate: r.publishedDate ?? undefined,
                 score: r.score ?? undefined,
+                imageUrl: r.img_src ?? r.thumbnail ?? undefined,
             });
             if (output.length >= count)
                 break;

package/dist/ee/domain-extractors.d.ts

@@ -1,48 +1,8 @@
 /**
  * Domain-aware structured extractors for WebPeel.
  *
- * When peel() fetches a URL that matches a known domain, the relevant
- * extractor fires and returns clean structured data + a markdown summary.
- *
- * Supported domains:
- *  - twitter.com / x.com  — tweets, threads, profiles
- *  - reddit.com            — posts with comments (via JSON API)
- *  - github.com            — repos, issues, PRs, users (via GitHub API)
- *  - news.ycombinator.com  — stories with comments (via HN Firebase API)
- */
-export interface DomainExtractResult {
-    /** Canonical domain name (e.g. 'twitter.com') */
-    domain: string;
-    /** Page type within the domain (e.g. 'tweet', 'thread', 'repo', 'issue') */
-    type: string;
-    /** Domain-specific structured data */
-    structured: Record<string, any>;
-    /** Clean markdown representation of the content */
-    cleanContent: string;
-    /** Raw HTML size in characters (from the actual HTML page fetched by the extractor) */
-    rawHtmlSize?: number;
-}
-/** An extractor receives the raw HTML and original URL, may make API calls. */
-export type DomainExtractor = (html: string, url: string) => Promise<DomainExtractResult | null>;
-/**
- * Returns the domain extractor for a URL, or null if none matches.
- */
-export declare function getDomainExtractor(url: string): DomainExtractor | null;
-/** Clear the extractor response cache (used in tests). */
-export declare function clearExtractorCache(): void;
-/**
- * Inject a Redis client for shared cross-pod caching.
- * Called from server startup after Redis is initialized.
- * Safe to call with null to disable Redis caching (e.g., CLI mode).
- */
-export declare function setExtractorRedis(redis: any): void;
-/**
- * Convenience: run the extractor for the URL (if one exists).
- * Wraps _extractDomainDataImpl with a two-tier cache:
- *   1. In-memory LRU (per-pod, fastest)
- *   2. Redis shared cache (cross-pod, shared across all replicas)
- *
- * With multiple API pods, Redis ensures the first pod to fetch a URL
- * populates cache for all others — eliminating redundant API calls.
+ * This file re-exports from individual extractor files for backward compatibility.
+ * Each extractor now lives in its own file under src/ee/extractors/.
  */
-export declare function extractDomainData(html: string, url: string): Promise<DomainExtractResult | null>;
+export { getDomainExtractor, hasDomainExtractor, extractDomainData, clearExtractorCache, setExtractorRedis, } from './extractors/index.js';
+export type { DomainExtractResult, DomainExtractor } from './extractors/index.js';