webpeel 0.21.83 → 0.21.85

package/dist/core/pipeline.d.ts

@@ -5,7 +5,7 @@
  * mutable PipelineContext.  The stages are called in order by peel().
  */
 import { type AutoScrollOptions } from './actions.js';
-import { type DomainExtractResult } from './domain-extractors-basic.js';
+import { type DomainExtractResult } from '../ee/domain-extractors.js';
 import { type ReadabilityResult } from './readability.js';
 import { type QuickAnswerResult } from './quick-answer.js';
 import { Timer } from './timing.js';

package/dist/core/pipeline.js

@@ -14,34 +14,8 @@ import { autoScroll as runAutoScroll } from './actions.js';
 import { extractStructured } from './extract.js';
 import { isPdfContentType, isDocxContentType, extractDocumentToFormat } from './documents.js';
 import { parseYouTubeUrl, getYouTubeTranscript } from './youtube.js';
-import { extractDomainDataBasic, getDomainExtractorBasic } from './domain-extractors-basic.js';
+import { extractDomainData, getDomainExtractor } from '../ee/domain-extractors.js';
 import { getDomainExtractHook, getDomainExtractorHook, getSPADomainsHook, getSPAPatternsHook } from './strategy-hooks.js';
-// ---------------------------------------------------------------------------
-// Domain extraction — lazy-load full extractors from compiled JS
-// ---------------------------------------------------------------------------
-// The compiled domain-extractors.js (312KB) ships in the npm package.
-// TypeScript source is NOT on GitHub (proprietary, .gitignore'd).
-// If compiled JS is missing (bare repo clone without proprietary files),
-// falls back to basic stub (no domain extraction, just standard markdown).
-// Server premium hooks can override for additional caching/intelligence.
-let _extractorsLoaded = false;
-let _extractDomainData = null;
-let _getDomainExtractor = null;
-async function loadExtractors() {
-    if (_extractorsLoaded)
-        return;
-    _extractorsLoaded = true;
-    try {
-        const mod = await import('./domain-extractors.js');
-        _extractDomainData = mod.extractDomainData;
-        _getDomainExtractor = mod.getDomainExtractor;
-    }
-    catch {
-        // Compiled JS not available (bare repo clone) — basic stub will be used
-    }
-}
-// Start loading immediately (non-blocking)
-loadExtractors();
 import { extractReadableContent } from './readability.js';
 import { quickAnswer as runQuickAnswer } from './quick-answer.js';
 import { Timer } from './timing.js';
@@ -56,28 +30,23 @@ const log = createLogger('pipeline');
 // ---------------------------------------------------------------------------
 /**
  * Check if a URL has a domain extractor.
- * Priority: premium hook → full extractors (repo/server) → basic stub.
+ * Priority: premium hook → ee/domain-extractors.
  */
 function hasDomainExtractor(url) {
     const hookFn = getDomainExtractorHook();
     if (hookFn)
         return hookFn(url) !== null;
-    if (_getDomainExtractor)
-        return _getDomainExtractor(url) !== null;
-    return getDomainExtractorBasic(url) !== null;
+    return getDomainExtractor(url) !== null;
 }
 /**
  * Run domain extraction on HTML/URL.
- * Priority: premium hook → compiled extractors → basic stub.
+ * Priority: premium hook → ee/domain-extractors.
  */
 async function runDomainExtract(html, url) {
     const hookFn = getDomainExtractHook();
     if (hookFn)
         return hookFn(html, url);
-    await loadExtractors();
-    if (_extractDomainData)
-        return _extractDomainData(html, url);
-    return extractDomainDataBasic(html, url);
+    return extractDomainData(html, url);
 }
 /** Create the initial PipelineContext with defaults */
 export function createContext(url, options) {
@@ -609,7 +578,7 @@ export async function fetchContent(ctx) {
         const canSolve = hasBrowserWorker || process.env.ENABLE_LOCAL_CHALLENGE_SOLVE === 'true';
         if (canSolve) {
             try {
-                const { solveChallenge } = await import('./challenge-solver.js');
+                const { solveChallenge } = await import('../ee/challenge-solver.js');
                 const { detectChallenge } = await import('./challenge-detection.js');
                 const rawHtml = fetchResult.html || '';
                 const detectionResult = detectChallenge(rawHtml, fetchResult.statusCode);
@@ -1179,7 +1148,7 @@ export async function postProcess(ctx) {
             const canSolve = hasBrowserWorker || process.env.ENABLE_LOCAL_CHALLENGE_SOLVE === 'true';
             if (canSolve && ctx.fetchResult?.html) {
                 try {
-                    const { solveChallenge } = await import('./challenge-solver.js');
+                    const { solveChallenge } = await import('../ee/challenge-solver.js');
                     const { detectChallenge } = await import('./challenge-detection.js');
                     const rawHtml = ctx.fetchResult.html;
                     const detectionResult = detectChallenge(rawHtml, ctx.fetchResult.statusCode);

package/dist/core/strategy-hooks.d.ts

@@ -10,7 +10,7 @@
  * All hook methods are optional — unset hooks are simply skipped.
  */
 import type { FetchResult } from './fetcher.js';
-import type { DomainExtractResult } from './domain-extractors-basic.js';
+import type { DomainExtractResult } from '../ee/domain-extractors.js';
 export interface StrategyResult extends FetchResult {
     method: 'simple' | 'browser' | 'stealth' | 'cached' | 'cloaked' | 'cycle' | 'peeltls' | 'cf-worker' | 'google-cache';
     /**

package/dist/index.d.ts

@@ -6,8 +6,7 @@
 import { cleanup, warmup, closePool, scrollAndWait, closeProfileBrowser } from './core/fetcher.js';
 import type { PeelOptions, PeelResult } from './types.js';
 export * from './types.js';
-export type { DomainExtractResult, DomainExtractor } from './core/domain-extractors-basic.js';
-export { getDomainExtractor, extractDomainData } from './core/domain-extractors-public.js';
+export { getDomainExtractor, extractDomainData, type DomainExtractResult, type DomainExtractor } from './ee/domain-extractors.js';
 export { crawl, type CrawlOptions, type CrawlResult, type CrawlProgress } from './core/crawler.js';
 export { discoverSitemap, type SitemapUrl, type SitemapResult } from './core/sitemap.js';
 export { mapDomain, type MapOptions, type MapResult } from './core/map.js';

package/dist/index.js

@@ -7,7 +7,9 @@ import { cleanup, warmup, closePool, scrollAndWait, closeProfileBrowser } from '
 import { createContext, normalizeOptions, handleYouTube, fetchContent, detectContentType, parseContent, postProcess, finalize, buildResult, } from './core/pipeline.js';
 import { checkUrlSafety } from './core/safe-browsing.js';
 export * from './types.js';
-export { getDomainExtractor, extractDomainData } from './core/domain-extractors-public.js';
+// Domain extractors — compiled JS ships in npm, TypeScript source is .gitignore'd.
+// Re-export types from the basic stub (always available), runtime functions via lazy wrapper.
+export { getDomainExtractor, extractDomainData } from './ee/domain-extractors.js';
 export { crawl } from './core/crawler.js';
 export { discoverSitemap } from './core/sitemap.js';
 export { mapDomain } from './core/map.js';

package/dist/server/app.js

@@ -18,6 +18,7 @@ import { createRateLimitMiddleware, RateLimiter } from './middleware/rate-limit.
 import { createHealthRouter } from './routes/health.js';
 import { createFetchRouter } from './routes/fetch.js';
 import { createSearchRouter } from './routes/search.js';
+import { createSmartSearchRouter } from './routes/smart-search.js';
 import { createUserRouter } from './routes/users.js';
 import { createStripeRouter, createBillingPortalRouter } from './routes/stripe.js';
 import { createOAuthRouter } from './routes/oauth.js';
@@ -55,19 +56,18 @@ import { requireScope } from './middleware/scope-guard.js';
 import { createCacheWarmRouter, startCacheWarmer } from './routes/cache-warm.js';
 import { warmup, cleanup as cleanupFetcher } from '../core/fetcher.js';
 // Proprietary modules — loaded dynamically so the build works without TypeScript source.
-// Compiled JS ships in npm/Docker. TypeScript source is .gitignore'd (not on GitHub).
 let setExtractorRedis;
 let registerPremiumHooks;
 try {
-    const de = await import('../core/domain-extractors.js');
+    const de = await import('../ee/domain-extractors.js');
     setExtractorRedis = de.setExtractorRedis;
 }
-catch { /* compiled JS not available */ }
+catch { /* ee module not available */ }
 try {
-    const ph = await import('./premium/index.js');
+    const ph = await import('../ee/premium-hooks.js');
     registerPremiumHooks = ph.registerPremiumHooks;
 }
-catch { /* compiled JS not available */ }
+catch { /* ee module not available */ }
 import { readFileSync } from 'fs';
 import { join, dirname } from 'path';
 import { fileURLToPath } from 'url';
@@ -315,6 +315,8 @@ export function createApp(config = {}) {
     app.use('/v1/screenshot', requireScope('full', 'read'));
     app.use(createScreenshotRouter(authStore));
     app.use(createSearchRouter(authStore));
+    // /v1/search/smart — intent detection + travel/commerce routing (POST)
+    app.use(createSmartSearchRouter(authStore));
     // /v1/research — lightweight research (search → fetch → compile), BYOK LLM optional
     app.use('/v1/research', requireScope('full', 'read'));
     app.use(createResearchRouter());

package/dist/server/routes/smart-search.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Smart Search endpoint — intent detection + travel/commerce routing
+ * POST /v1/search/smart
+ *
+ * Detects user intent from natural language and routes to the best source:
+ * - cars       → Cars.com with browser rendering + structured extraction
+ * - flights    → Google Flights with browser rendering + flight extractor
+ * - hotels     → Google Hotels with browser rendering
+ * - rental     → Kayak with browser rendering + rental extractor
+ * - restaurants → Yelp Fusion API extractor
+ * - general    → SearXNG with smart enrichment (peel() for top 2)
+ */
+import { Router } from 'express';
+import '../types.js';
+import { AuthStore } from '../auth-store.js';
+export interface SearchIntent {
+    type: 'cars' | 'flights' | 'hotels' | 'rental' | 'restaurants' | 'general';
+    query: string;
+    params: Record<string, string>;
+}
+export interface SmartSearchResult {
+    type: 'cars' | 'flights' | 'hotels' | 'rental' | 'restaurants' | 'general';
+    source: string;
+    sourceUrl: string;
+    content: string;
+    title?: string;
+    domainData?: any;
+    structured?: any;
+    results?: any[];
+    tokens: number;
+    fetchTimeMs: number;
+    loadingMessage?: string;
+}
+export declare function detectSearchIntent(query: string): SearchIntent;
+export declare function createSmartSearchRouter(authStore: AuthStore): Router;

package/dist/server/routes/smart-search.js

@@ -0,0 +1,358 @@
+/**
+ * Smart Search endpoint — intent detection + travel/commerce routing
+ * POST /v1/search/smart
+ *
+ * Detects user intent from natural language and routes to the best source:
+ * - cars       → Cars.com with browser rendering + structured extraction
+ * - flights    → Google Flights with browser rendering + flight extractor
+ * - hotels     → Google Hotels with browser rendering
+ * - rental     → Kayak with browser rendering + rental extractor
+ * - restaurants → Yelp Fusion API extractor
+ * - general    → SearXNG with smart enrichment (peel() for top 2)
+ */
+import { Router } from 'express';
+import '../types.js'; // Augments Express.Request with requestId, auth
+import { peel } from '../../index.js';
+import { getBestSearchProvider, } from '../../core/search-provider.js';
+import { getSourceCredibility } from '../../core/source-credibility.js';
+// ─── Intent Detection ──────────────────────────────────────────────────────
+export function detectSearchIntent(query) {
+    const q = query.toLowerCase();
+    // Cars: vehicle name/type + buying signals
+    if (/\b(car|cars|vehicle|sedan|suv|truck|honda|toyota|tesla|bmw|ford|chevy|chevrolet|nissan|hyundai|kia|mazda|subaru|lexus|audi|mercedes|volkswagen|jeep|dodge|ram|buick|cadillac|gmc|chrysler|acura|infiniti|volvo|porsche|mini|fiat|mitsubishi)\b/.test(q) &&
+        /\b(buy|cheap|under|budget|price|used|new|for sale|listing|deal)\b/.test(q)) {
+        const priceMatch = q.match(/(?:under|\$|budget|max)\s*\$?(\d[\d,]*)/);
+        const priceValue = priceMatch ? priceMatch[1].replace(/,/g, '') : '';
+        // Find all 5-digit numbers, pick the one that isn't the price
+        const allZips = [...q.matchAll(/\b(\d{5})\b/g)].map(m => m[1]);
+        const finalZip = allZips.find(z => z !== priceValue) || '10001';
+        return {
+            type: 'cars',
+            query: q,
+            params: {
+                maxPrice: priceValue,
+                zip: finalZip,
+            },
+        };
+    }
+    // Flights: "flight", "fly", city-to-city patterns with dates
+    if (/\b(flight|flights|fly|flying|airline|plane)\b/.test(q) ||
+        (/\b(from|to)\b.*\b(to|from)\b/.test(q) && /\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec|\d{1,2}\/\d{1,2})\b/.test(q))) {
+        return { type: 'flights', query: q, params: {} };
+    }
+    // Hotels: "hotel", "stay", "accommodation", etc. + location signal
+    if (/\b(hotel|hotels|motel|stay|accommodation|lodging|inn|resort|airbnb|hostel)\b/.test(q) &&
+        /\b(in|near|at|around|cheap|best|book)\b/.test(q)) {
+        return { type: 'hotels', query: q, params: {} };
+    }
+    // Car rental: "rent a car", "car rental", "rental car"
+    if (/\b(rent|rental)\b.*\b(car|vehicle|suv)\b/.test(q) ||
+        /\bcar\s+rental\b/.test(q)) {
+        return { type: 'rental', query: q, params: {} };
+    }
+    // Restaurants: food/dining + location/quality signal
+    if (/\b(restaurant|restaurants|food|eat|dinner|lunch|pizza|sushi|burger|cafe|bar|bistro|brunch|breakfast)\b/.test(q) &&
+        /\b(in|near|best|top|good|cheap)\b/.test(q)) {
+        return { type: 'restaurants', query: q, params: {} };
+    }
+    return { type: 'general', query: q, params: {} };
+}
+// ─── Intent Handlers ───────────────────────────────────────────────────────
+async function handleCarSearch(intent) {
+    const t0 = Date.now();
+    // Build a clean keyword: strip the common car/buy/deal words to surface the actual vehicle name
+    const keyword = intent.query
+        .replace(/\b(buy|cheap|under|budget|price|used|new|for sale|listing|deal|car|cars)\b/gi, '')
+        .replace(/\s+/g, ' ')
+        .trim();
+    const params = new URLSearchParams({
+        keyword,
+        sort: 'list_price',
+        stock_type: 'all',
+        zip: intent.params.zip || '10001',
+        maximum_distance: '50',
+    });
+    if (intent.params.maxPrice)
+        params.set('list_price_max', intent.params.maxPrice);
+    const url = `https://www.cars.com/shopping/results/?${params.toString()}`;
+    try {
+        const result = await peel(url, { render: true, timeout: 25000 });
+        return {
+            type: 'cars',
+            source: 'Cars.com',
+            sourceUrl: url,
+            content: result.content,
+            title: result.title,
+            domainData: result.domainData,
+            structured: result.domainData?.structured,
+            tokens: result.tokens,
+            fetchTimeMs: Date.now() - t0,
+        };
+    }
+    catch (err) {
+        throw new Error(`Cars.com search failed: ${err.message}`);
+    }
+}
+async function handleFlightSearch(intent) {
+    const t0 = Date.now();
+    const gfUrl = `https://www.google.com/travel/flights?q=Flights+${encodeURIComponent(intent.query)}+one+way`;
+    try {
+        const result = await peel(gfUrl, { render: true, timeout: 30000 });
+        return {
+            type: 'flights',
+            source: 'Google Flights',
+            sourceUrl: gfUrl,
+            content: result.content,
+            title: result.title,
+            domainData: result.domainData,
+            structured: result.domainData?.structured,
+            tokens: result.tokens,
+            fetchTimeMs: Date.now() - t0,
+        };
+    }
+    catch (err) {
+        throw new Error(`Google Flights search failed: ${err.message}`);
+    }
+}
+async function handleHotelSearch(intent) {
+    const t0 = Date.now();
+    const ghUrl = `https://www.google.com/travel/hotels?q=${encodeURIComponent(intent.query)}`;
+    try {
+        const result = await peel(ghUrl, { render: true, timeout: 30000 });
+        return {
+            type: 'hotels',
+            source: 'Google Hotels',
+            sourceUrl: ghUrl,
+            content: result.content,
+            title: result.title,
+            domainData: result.domainData,
+            structured: result.domainData?.structured,
+            tokens: result.tokens,
+            fetchTimeMs: Date.now() - t0,
+        };
+    }
+    catch (err) {
+        throw new Error(`Google Hotels search failed: ${err.message}`);
+    }
+}
+async function handleRentalSearch(intent) {
+    const t0 = Date.now();
+    // Build Kayak car rental URL: /cars/<location>/<date-range>
+    // For simplicity, use a search-style URL that will browser-render fine
+    const encodedQuery = encodeURIComponent(intent.query.replace(/\b(rent|rental|car|a|vehicle|suv)\b/gi, '').trim() || intent.query);
+    const kayakUrl = `https://www.kayak.com/cars/${encodedQuery}/2025-04-10/2025-04-13/`;
+    try {
+        const result = await peel(kayakUrl, { render: true, timeout: 30000 });
+        return {
+            type: 'rental',
+            source: 'Kayak',
+            sourceUrl: kayakUrl,
+            content: result.content,
+            title: result.title,
+            domainData: result.domainData,
+            structured: result.domainData?.structured,
+            tokens: result.tokens,
+            fetchTimeMs: Date.now() - t0,
+        };
+    }
+    catch (err) {
+        throw new Error(`Kayak car rental search failed: ${err.message}`);
+    }
+}
+async function handleRestaurantSearch(intent) {
+    const t0 = Date.now();
+    const yelpUrl = `https://www.yelp.com/search?find_desc=${encodeURIComponent(intent.query.replace(/\b(best|top|good|cheap|near me)\b/gi, '').trim())}&find_loc=${encodeURIComponent('New York, NY')}`;
+    try {
+        const result = await peel(yelpUrl, { render: true, timeout: 25000 });
+        return {
+            type: 'restaurants',
+            source: 'Yelp',
+            sourceUrl: yelpUrl,
+            content: result.content,
+            title: result.title,
+            domainData: result.domainData,
+            structured: result.domainData?.structured,
+            tokens: result.tokens,
+            fetchTimeMs: Date.now() - t0,
+        };
+    }
+    catch (err) {
+        throw new Error(`Yelp search failed: ${err.message}`);
+    }
+}
+async function handleGeneralSearch(query) {
+    const t0 = Date.now();
+    const { provider: searchProvider } = getBestSearchProvider();
+    const rawResults = await searchProvider.searchWeb(query, { count: 10 });
+    const getDomain = (url) => {
+        try {
+            return new URL(url).hostname.replace(/^www\./, '');
+        }
+        catch {
+            return '';
+        }
+    };
+    const tierOrder = { official: 0, established: 1, community: 2, new: 3, suspicious: 4 };
+    let results = rawResults
+        .map((r) => {
+        const cred = getSourceCredibility(r.url);
+        return {
+            title: r.title,
+            url: r.url,
+            snippet: r.snippet,
+            domain: getDomain(r.url),
+            credibility: cred,
+        };
+    })
+        .sort((a, b) => {
+        const aTier = tierOrder[a.credibility?.tier || 'new'] ?? 3;
+        const bTier = tierOrder[b.credibility?.tier || 'new'] ?? 3;
+        return aTier - bTier;
+    })
+        .map((r, i) => ({ ...r, rank: i + 1 }));
+    // Enrich top 2 results with peel() for richer content
+    const top2 = results.slice(0, 2);
+    const enriched = await Promise.allSettled(top2.map(async (r) => {
+        try {
+            const peeled = await peel(r.url, { render: true, timeout: 15000, maxTokens: 2000 });
+            return { url: r.url, content: peeled.content?.substring(0, 1500), fetchTimeMs: peeled.elapsed };
+        }
+        catch {
+            return { url: r.url, content: null, fetchTimeMs: 0 };
+        }
+    }));
+    for (const settled of enriched) {
+        if (settled.status === 'fulfilled' && settled.value.content) {
+            const match = results.find((r) => r.url === settled.value.url);
+            if (match) {
+                match.content = settled.value.content;
+                match.fetchTimeMs = settled.value.fetchTimeMs;
+            }
+        }
+    }
+    const content = results
+        .map((r, i) => `${i + 1}. **${r.title}**\n   ${r.url}\n   ${r.snippet}`)
+        .join('\n\n');
+    return {
+        type: 'general',
+        source: 'Web Search',
+        sourceUrl: `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`,
+        content,
+        results,
+        tokens: content.split(/\s+/).length,
+        fetchTimeMs: Date.now() - t0,
+    };
+}
+// ─── Loading message by intent type ────────────────────────────────────────
+function getLoadingMessage(type) {
+    const msgs = {
+        cars: 'Searching cars on Cars.com…',
+        flights: 'Finding flights on Google Flights…',
+        hotels: 'Looking up hotels on Google Hotels…',
+        rental: 'Searching rental cars on Kayak…',
+        restaurants: 'Finding restaurants on Yelp…',
+        general: 'Searching the web…',
+    };
+    return msgs[type] || 'Searching…';
+}
+// ─── Router ────────────────────────────────────────────────────────────────
+export function createSmartSearchRouter(authStore) {
+    const router = Router();
+    router.post('/v1/search/smart', async (req, res) => {
+        try {
+            // Require authentication
+            const authId = req.auth?.keyInfo?.accountId || req.user?.userId;
+            if (!authId) {
+                res.status(401).json({
+                    success: false,
+                    error: {
+                        type: 'authentication_required',
+                        message: 'API key required. Get one free at https://app.webpeel.dev',
+                        docs: 'https://webpeel.dev/docs/api-reference#authentication',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            const { q, location, zip } = req.body;
+            if (!q || typeof q !== 'string' || !q.trim()) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Missing or invalid "q" field in request body',
+                        hint: 'POST /v1/search/smart with JSON body: { "q": "your search query" }',
+                        docs: 'https://webpeel.dev/docs/api-reference#smart-search',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            const query = q.trim();
+            const intent = detectSearchIntent(query);
+            // Override zip from request body if provided
+            if (zip && intent.params) {
+                intent.params.zip = zip;
+            }
+            // Also try to extract location context from query if "location" is provided
+            if (location && intent.type === 'restaurants') {
+                // Will be passed in URL construction
+                intent.location = location;
+            }
+            let smartResult;
+            switch (intent.type) {
+                case 'cars':
+                    smartResult = await handleCarSearch(intent);
+                    break;
+                case 'flights':
+                    smartResult = await handleFlightSearch(intent);
+                    break;
+                case 'hotels':
+                    smartResult = await handleHotelSearch(intent);
+                    break;
+                case 'rental':
+                    smartResult = await handleRentalSearch(intent);
+                    break;
+                case 'restaurants':
+                    smartResult = await handleRestaurantSearch(intent);
+                    break;
+                default:
+                    smartResult = await handleGeneralSearch(query);
+            }
+            // Add loading message hint for frontend UX
+            smartResult.loadingMessage = getLoadingMessage(intent.type);
+            // Track usage
+            const pgStore = authStore;
+            if (req.auth?.keyInfo?.key && typeof pgStore.trackUsage === 'function') {
+                if (typeof pgStore.trackBurstUsage === 'function') {
+                    await pgStore.trackBurstUsage(req.auth.keyInfo.key);
+                }
+                if (!req.auth?.softLimited) {
+                    await pgStore.trackUsage(req.auth.keyInfo.key, 'smart-search');
+                }
+            }
+            res.setHeader('X-Intent-Type', intent.type);
+            res.setHeader('X-Source', smartResult.source);
+            res.setHeader('X-Processing-Time', smartResult.fetchTimeMs.toString());
+            res.setHeader('Cache-Control', 'no-store');
+            res.json({
+                success: true,
+                data: smartResult,
+            });
+        }
+        catch (error) {
+            const err = error;
+            console.error('Smart search error:', err.message, err.stack);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'smart_search_failed',
+                    message: err.message || 'Smart search failed. Please try again.',
+                    docs: 'https://webpeel.dev/docs/api-reference#smart-search',
+                },
+                requestId: req.requestId,
+            });
+        }
+    });
+    return router;
+}

package/dist/types.d.ts

@@ -309,7 +309,7 @@ export interface PeelResult {
      */
     readability?: import('./core/readability.js').ReadabilityResult;
     /** Domain-aware structured data (Twitter, Reddit, GitHub, HN). Present when URL matches a known domain. */
-    domainData?: import('./core/domain-extractors-basic.js').DomainExtractResult;
+    domainData?: import('./ee/domain-extractors.js').DomainExtractResult;
     /** Quick answer result (when question option is set). BM25-powered, no LLM needed. */
     quickAnswer?: import('./core/quick-answer.js').QuickAnswerResult;
     /** Per-stage timing breakdown in milliseconds. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.83",
+  "version": "0.21.85",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",