webpeel 0.21.66 → 0.21.68

package/dist/core/domain-extractors.d.ts

@@ -28,8 +28,21 @@ export type DomainExtractor = (html: string, url: string) => Promise<DomainExtra
  * Returns the domain extractor for a URL, or null if none matches.
  */
 export declare function getDomainExtractor(url: string): DomainExtractor | null;
+/** Clear the extractor response cache (used in tests). */
+export declare function clearExtractorCache(): void;
+/**
+ * Inject a Redis client for shared cross-pod caching.
+ * Called from server startup after Redis is initialized.
+ * Safe to call with null to disable Redis caching (e.g., CLI mode).
+ */
+export declare function setExtractorRedis(redis: any): void;
 /**
  * Convenience: run the extractor for the URL (if one exists).
- * Returns null when no extractor matches or extraction fails.
+ * Wraps _extractDomainDataImpl with a two-tier cache:
+ *   1. In-memory LRU (per-pod, fastest)
+ *   2. Redis shared cache (cross-pod, shared across all replicas)
+ *
+ * With multiple API pods, Redis ensures the first pod to fetch a URL
+ * populates cache for all others — eliminating redundant API calls.
  */
 export declare function extractDomainData(html: string, url: string): Promise<DomainExtractResult | null>;

package/dist/core/domain-extractors.js

@@ -112,6 +112,11 @@ const REGISTRY = [
     { match: (h) => h === 'open-meteo.com' || h === 'api.open-meteo.com' || h === 'www.open-meteo.com', extractor: weatherExtractor },
     { match: (h) => h === 'weather.com' || h === 'www.weather.com', extractor: weatherExtractor },
     { match: (h) => h === 'accuweather.com' || h === 'www.accuweather.com', extractor: weatherExtractor },
+    // ── Marketplaces & Shopping ───────────────────────────────────────────────
+    { match: (h) => h === 'facebook.com' || h === 'www.facebook.com', extractor: facebookMarketplaceExtractor },
+    { match: (h) => h === 'etsy.com' || h === 'www.etsy.com', extractor: etsyExtractor },
+    { match: (h) => h === 'cars.com' || h === 'www.cars.com', extractor: carsComExtractor },
+    { match: (h) => h === 'ebay.com' || h === 'www.ebay.com', extractor: ebayExtractor },
 ];
 /**
  * Returns the domain extractor for a URL, or null if none matches.
@@ -131,11 +136,84 @@ export function getDomainExtractor(url) {
     }
     return null;
 }
+// ── Extractor Response Cache ──────────────────────────────────────────────
+// Caches successful API responses for 5 minutes to survive rate limits.
+// If the API rate-limits on the next request, we serve from cache instead
+// of falling back to garbage browser rendering (cookie walls, "Loading…").
+// Key: normalized URL (no query/hash), Value: { result, timestamp }
+const EXTRACTOR_CACHE = new Map();
+/** Clear the extractor response cache (used in tests). */
+export function clearExtractorCache() { EXTRACTOR_CACHE.clear(); }
+const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
+function getCachedExtractorResult(url) {
+    const key = url.replace(/[?#].*$/, '').toLowerCase(); // strip query+hash
+    const entry = EXTRACTOR_CACHE.get(key);
+    if (entry && Date.now() - entry.ts < CACHE_TTL_MS) {
+        return entry.result;
+    }
+    EXTRACTOR_CACHE.delete(key); // expired — evict
+    return null;
+}
+function setCachedExtractorResult(url, result) {
+    const key = url.replace(/[?#].*$/, '').toLowerCase();
+    EXTRACTOR_CACHE.set(key, { result, ts: Date.now() });
+    // Keep cache size bounded at 500 entries (evict oldest)
+    if (EXTRACTOR_CACHE.size > 500) {
+        const oldest = EXTRACTOR_CACHE.keys().next().value;
+        if (oldest)
+            EXTRACTOR_CACHE.delete(oldest);
+    }
+}
+// ── Redis Shared Cache (cross-pod cache for multi-replica deployments) ────────
+// When running multiple API pods, each pod has its own in-memory cache.
+// With 6 pods, 5/6 requests for the same URL miss cache.
+// Redis solves this: all pods share one cache, so the first pod to fetch
+// populates it for all others.
+//
+// Redis is injected from the server startup to keep this core module
+// dependency-free (works in CLI mode without Redis too).
+let _redisClient = null;
+const REDIS_CACHE_PREFIX = 'wp:ext:';
+const REDIS_CACHE_TTL_SECS = 300; // 5 minutes
 /**
- * Convenience: run the extractor for the URL (if one exists).
+ * Inject a Redis client for shared cross-pod caching.
+ * Called from server startup after Redis is initialized.
+ * Safe to call with null to disable Redis caching (e.g., CLI mode).
+ */
+export function setExtractorRedis(redis) {
+    _redisClient = redis;
+}
+async function getRedisCache(url) {
+    try {
+        if (!_redisClient)
+            return null;
+        const key = REDIS_CACHE_PREFIX + url.replace(/[?#].*$/, '').toLowerCase();
+        const cached = await _redisClient.get(key);
+        if (!cached)
+            return null;
+        return JSON.parse(cached);
+    }
+    catch {
+        return null; // Redis unavailable — fall back to in-memory cache
+    }
+}
+async function setRedisCache(url, result) {
+    try {
+        if (!_redisClient)
+            return;
+        const key = REDIS_CACHE_PREFIX + url.replace(/[?#].*$/, '').toLowerCase();
+        await _redisClient.set(key, JSON.stringify(result), 'EX', REDIS_CACHE_TTL_SECS);
+    }
+    catch {
+        // Redis unavailable — in-memory cache still works, this is non-fatal
+    }
+}
+// ─────────────────────────────────────────────────────────────────────────────
+/**
+ * Internal implementation: run the extractor for the URL (if one exists).
  * Returns null when no extractor matches or extraction fails.
  */
-export async function extractDomainData(html, url) {
+async function _extractDomainDataImpl(html, url) {
     const extractor = getDomainExtractor(url);
     if (!extractor)
         return null;
@@ -146,6 +224,43 @@ export async function extractDomainData(html, url) {
         return null;
     }
 }
+/**
+ * Convenience: run the extractor for the URL (if one exists).
+ * Wraps _extractDomainDataImpl with a two-tier cache:
+ *   1. In-memory LRU (per-pod, fastest)
+ *   2. Redis shared cache (cross-pod, shared across all replicas)
+ *
+ * With multiple API pods, Redis ensures the first pod to fetch a URL
+ * populates cache for all others — eliminating redundant API calls.
+ */
+export async function extractDomainData(html, url) {
+    // 1. Check in-memory cache (fastest — no network)
+    const cached = getCachedExtractorResult(url);
+    if (cached)
+        return cached;
+    // 2. Check Redis cache (shared across all pods)
+    const redisCached = await getRedisCache(url);
+    if (redisCached) {
+        // Populate local in-memory cache to avoid Redis round-trips on repeat
+        setCachedExtractorResult(url, redisCached);
+        return redisCached;
+    }
+    // 3. Try the real extractor
+    const result = await _extractDomainDataImpl(html, url);
+    if (result && result.cleanContent.length > 20) {
+        // 4. Cache the successful result in both layers
+        setCachedExtractorResult(url, result);
+        void setRedisCache(url, result); // fire-and-forget, non-blocking
+        return result;
+    }
+    // 5. Extractor failed/returned garbage — check for any stale cache entry
+    //    (stale structured data beats a browser "Loading…" page)
+    const stale = getCachedExtractorResult(url);
+    if (stale)
+        return stale;
+    // 6. Genuinely nothing — return null so the pipeline falls back to fetch
+    return result;
+}
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
@@ -184,6 +299,13 @@ async function fetchJson(url, customHeaders) {
             redirect: 'follow',
         });
         clearTimeout(timer);
+        // Surface 429 as a thrown error so callers can detect rate-limiting
+        // and the cache wrapper can serve stale results instead of garbage.
+        if (resp.status === 429) {
+            const err = new Error(`429 Too Many Requests: ${url}`);
+            err.statusCode = 429;
+            throw err;
+        }
         const text = await resp.text();
         const parsed = tryParseJson(text);
         if (parsed === null && text.length > 0) {
@@ -4784,3 +4906,281 @@ async function weatherExtractor(_html, url) {
         return null;
     }
 }
+// ---------------------------------------------------------------------------
+// Facebook Marketplace extractor (login-wall fallback)
+// ---------------------------------------------------------------------------
+async function facebookMarketplaceExtractor(_html, url) {
+    const u = new URL(url);
+    if (!u.pathname.includes('/marketplace'))
+        return null;
+    const query = u.searchParams.get('query') || '';
+    const maxPrice = u.searchParams.get('maxPrice') || '';
+    const minPrice = u.searchParams.get('minPrice') || '';
+    // Extract location segment: /marketplace/nyc/search → "nyc"
+    const locationMatch = u.pathname.match(/\/marketplace\/([^/]+)(?:\/|$)/);
+    const location = (locationMatch?.[1] && locationMatch[1] !== 'search' && locationMatch[1] !== 'category') ? locationMatch[1] : '';
+    const priceRange = [minPrice && `$${minPrice}`, maxPrice && `$${maxPrice}`].filter(Boolean).join(' – ');
+    const lines = [
+        `# 🛒 Facebook Marketplace`,
+        '',
+        `**Search:** ${query || 'Browse all'}`,
+        ...(location ? [`**Location:** ${location}`] : []),
+        ...(priceRange ? [`**Price range:** ${priceRange}`] : []),
+        '',
+        '> ⚠️ Facebook Marketplace requires authentication. WebPeel cannot access listings directly.',
+        '',
+        '**Alternative searches that work:**',
+    ];
+    if (query) {
+        const clUrl = `https://newyork.craigslist.org/search/sss?query=${encodeURIComponent(query)}${maxPrice ? '&max_price=' + maxPrice : ''}`;
+        const carsUrl = `https://www.cars.com/shopping/results/?keyword=${encodeURIComponent(query)}&list_price_max=${maxPrice || ''}&zip=10001&stock_type=used`;
+        const ebayUrl = `https://www.ebay.com/sch/i.html?_nkw=${encodeURIComponent(query)}${maxPrice ? '&_udhi=' + maxPrice : ''}&LH_BIN=1`;
+        lines.push(`- \`webpeel "${clUrl}"\` — Craigslist`, `- \`webpeel "${carsUrl}"\` — Cars.com`, `- \`webpeel "${ebayUrl}"\` — eBay`);
+    }
+    lines.push('', '*Tip: Craigslist and Cars.com return full structured results with WebPeel.*');
+    return {
+        domain: 'facebook.com',
+        type: 'blocked',
+        structured: {
+            query,
+            location,
+            minPrice,
+            maxPrice,
+            reason: 'authentication required',
+            alternatives: ['craigslist', 'cars.com', 'ebay'],
+        },
+        cleanContent: lines.join('\n'),
+    };
+}
+// ---------------------------------------------------------------------------
+// Etsy extractor (bot-block fallback with Google site-search suggestion)
+// ---------------------------------------------------------------------------
+async function etsyExtractor(_html, url) {
+    const u = new URL(url);
+    // Extract search query from various URL patterns
+    // /search?q=handmade+jewelry  OR  /search/handmade-jewelry
+    let query = u.searchParams.get('q') || '';
+    if (!query) {
+        const pathMatch = u.pathname.match(/\/search\/([^?#]+)/);
+        if (pathMatch)
+            query = decodeURIComponent(pathMatch[1].replace(/-/g, ' '));
+    }
+    // Shop page: /shop/ShopName
+    const shopMatch = u.pathname.match(/^\/shop\/([^/?#]+)/);
+    const shopName = shopMatch?.[1] || '';
+    if (!query && !shopName)
+        return null;
+    const googleUrl = query
+        ? `https://www.google.com/search?q=site:etsy.com+${encodeURIComponent(query)}`
+        : `https://www.google.com/search?q=site:etsy.com+${encodeURIComponent(shopName)}`;
+    const etsySearchUrl = query ? `https://www.etsy.com/search?q=${encodeURIComponent(query)}` : url;
+    const displayTitle = query ? `"${query}"` : `Shop: ${shopName}`;
+    const cleanContent = [
+        `# 🎨 Etsy — ${displayTitle}`,
+        '',
+        '> ⚠️ Etsy blocks automated access. WebPeel cannot scrape listings directly.',
+        '',
+        '**Alternatives that work:**',
+        `- \`webpeel "${googleUrl}"\` — Google site:etsy.com results`,
+        `- Direct link: [etsy.com/search?q=${encodeURIComponent(query || shopName)}](${etsySearchUrl})`,
+        '',
+        ...(query ? [
+            '**Similar items on open marketplaces:**',
+            `- \`webpeel "https://www.ebay.com/sch/i.html?_nkw=${encodeURIComponent(query)}&LH_BIN=1"\` — eBay`,
+            `- \`webpeel "https://newyork.craigslist.org/search/sss?query=${encodeURIComponent(query)}"\` — Craigslist`,
+        ] : []),
+        '',
+        '*Etsy Open API v3 (free key at etsy.com/developers) can unlock direct access.*',
+    ].join('\n');
+    return {
+        domain: 'etsy.com',
+        type: 'blocked',
+        structured: {
+            query,
+            shopName,
+            reason: 'bot-block',
+            googleFallback: googleUrl,
+        },
+        cleanContent,
+    };
+}
+// ---------------------------------------------------------------------------
+// Cars.com extractor — structured parsing via data-vehicle-details JSON attrs
+// ---------------------------------------------------------------------------
+async function carsComExtractor(html, url) {
+    try {
+        const { load } = await import('cheerio');
+        const $ = load(html);
+        const u = new URL(url);
+        const keyword = u.searchParams.get('keyword') || '';
+        const maxPrice = u.searchParams.get('list_price_max') || '';
+        const minPrice = u.searchParams.get('list_price_min') || '';
+        const zip = u.searchParams.get('zip') || '';
+        const stockType = u.searchParams.get('stock_type') || '';
+        // Individual vehicle detail page
+        if (u.pathname.includes('/vehicledetail/')) {
+            const title = $('h1').first().text().trim() ||
+                $('title').text().trim().split(' | ')[0];
+            if (!title)
+                return null;
+            const price = $('[class*="price"]').first().text().trim();
+            const mileage = $('[class*="mileage"]').first().text().trim();
+            return {
+                domain: 'cars.com',
+                type: 'listing',
+                structured: { title, price, mileage, url },
+                cleanContent: [
+                    `# 🚗 ${title}`,
+                    price && `**Price:** ${price}`,
+                    mileage && `**Mileage:** ${mileage}`,
+                    `\n[View listing](${url})`,
+                ].filter(Boolean).join('\n'),
+            };
+        }
+        // Search results page — Cars.com embeds JSON in fuse-card data-vehicle-details
+        const listings = [];
+        $('fuse-card[data-vehicle-details]').each((_, el) => {
+            try {
+                const raw = $(el).attr('data-vehicle-details');
+                if (!raw)
+                    return;
+                const v = JSON.parse(raw);
+                const listingId = v.listingId || $(el).attr('data-listing-id') || '';
+                const cardLink = $(el).find('card-gallery').attr('card-link') || (listingId ? `/vehicledetail/${listingId}/` : '');
+                const title = `${v.stockType || 'Used'} ${v.year} ${v.make} ${v.model}${v.trim ? ' ' + v.trim : ''}`.trim();
+                const price = v.price ? `$${Number(v.price).toLocaleString()}` : '';
+                const mileage = v.mileage ? `${Number(v.mileage).toLocaleString()} mi` : '';
+                const bodyStyle = v.bodyStyle || '';
+                const fuelType = v.fuelType || '';
+                if (title && title !== 'Used  ') {
+                    listings.push({ title, price, mileage, bodyStyle, fuelType, url: cardLink });
+                }
+            }
+            catch { /* skip malformed */ }
+        });
+        if (listings.length === 0)
+            return null; // Let pipeline handle it
+        const priceRange = [minPrice && `$${minPrice}`, maxPrice && `$${maxPrice}`].filter(Boolean).join(' – ');
+        const header = [
+            `# 🚗 Cars.com — ${keyword || 'Vehicle Search'}`,
+            '',
+            keyword && `**Search:** ${keyword}`,
+            zip && `**Location:** ZIP ${zip}`,
+            priceRange && `**Price:** up to $${maxPrice}`,
+            stockType && `**Stock:** ${stockType}`,
+            `**Results:** ${listings.length} listings`,
+            '',
+        ].filter(Boolean).join('\n');
+        const rows = listings.slice(0, 20).map((l, i) => {
+            const parts = [
+                `${i + 1}. **${l.title}**`,
+                l.price,
+                l.mileage,
+                l.bodyStyle,
+                l.url && `[→](https://www.cars.com${l.url})`,
+            ].filter(Boolean);
+            return parts.join(' · ');
+        });
+        return {
+            domain: 'cars.com',
+            type: 'search',
+            structured: { keyword, zip, minPrice, maxPrice, stockType, count: listings.length, listings },
+            cleanContent: header + rows.join('\n'),
+        };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'Cars.com extractor error:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}
+// ---------------------------------------------------------------------------
+// eBay extractor — clean up noisy search results
+// ---------------------------------------------------------------------------
+async function ebayExtractor(html, url) {
+    try {
+        const { load } = await import('cheerio');
+        const $ = load(html);
+        const u = new URL(url);
+        // Individual item page
+        if (u.pathname.startsWith('/itm/')) {
+            const title = $('h1').first().text().trim();
+            if (!title)
+                return null;
+            const price = $('[class*="price"]').not('[class*="shipping"]').first().text().trim();
+            const condition = $('[class*="condition"]').first().text().trim();
+            return {
+                domain: 'ebay.com',
+                type: 'listing',
+                structured: { title, price, condition, url },
+                cleanContent: [
+                    `# 🛍 ${title}`,
+                    price && `**Price:** ${price}`,
+                    condition && `**Condition:** ${condition}`,
+                    `\n[View on eBay](${url})`,
+                ].filter(Boolean).join('\n'),
+            };
+        }
+        // Search results page
+        const keyword = u.searchParams.get('_nkw') || '';
+        const maxPrice = u.searchParams.get('_udhi') || '';
+        const minPrice = u.searchParams.get('_udlo') || '';
+        const listings = [];
+        // eBay search results use li[data-listingid] + .s-card__title / .s-card__price
+        $('li[data-listingid]').each((_, el) => {
+            const titleRaw = $(el).find('.s-card__title').text().trim()
+                .replace(/Opens in a new window or tab/g, '')
+                .replace(/^New Listing\s*/i, '')
+                .trim();
+            if (!titleRaw || titleRaw === 'Shop on eBay')
+                return;
+            const title = titleRaw;
+            const price = $(el).find('.s-card__price').first().text().trim();
+            // .s-card__subtitle contains "DealerNameCondition" as merged text — extract condition keyword
+            const subtitleText = $(el).find('.s-card__subtitle').text().trim();
+            const conditionKeywords = ['Pre-Owned', 'Brand New', 'Open Box', 'Refurbished', 'For Parts'];
+            const condition = conditionKeywords.find((k) => subtitleText.includes(k)) || '';
+            // Get clean URL — extract /itm/<id> and strip tracking params
+            let href = '';
+            const itemLink = $(el).find('a[href*="/itm/"]').first().attr('href') || '';
+            const itmMatch = itemLink.match(/(https?:\/\/[^/]*\/itm\/\d+)/);
+            if (itmMatch)
+                href = itmMatch[1];
+            const listingId = $(el).attr('data-listingid') || '';
+            if (!href && listingId)
+                href = `https://www.ebay.com/itm/${listingId}`;
+            listings.push({ title, price, condition, url: href });
+        });
+        if (listings.length === 0)
+            return null; // Let pipeline handle it
+        const priceRange = [minPrice && `$${minPrice}`, maxPrice && `$${maxPrice}`].filter(Boolean).join(' – ');
+        const header = [
+            `# 🛍 eBay — ${keyword || 'Search Results'}`,
+            '',
+            keyword && `**Search:** ${keyword}`,
+            priceRange && `**Price:** up to $${maxPrice}`,
+            `**Results:** ${listings.length} listings`,
+            '',
+        ].filter(Boolean).join('\n');
+        const rows = listings.slice(0, 20).map((l, i) => {
+            const parts = [
+                `${i + 1}. **${l.title}**`,
+                l.price,
+                l.condition && `[${l.condition}]`,
+                l.url && `[→](${l.url})`,
+            ].filter(Boolean);
+            return parts.join(' · ');
+        });
+        return {
+            domain: 'ebay.com',
+            type: 'search',
+            structured: { keyword, minPrice, maxPrice, count: listings.length, listings },
+            cleanContent: header + rows.join('\n'),
+        };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'eBay extractor error:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}

package/dist/server/app.js

@@ -54,6 +54,7 @@ import { createSentryHooks } from './sentry.js';
 import { requireScope } from './middleware/scope-guard.js';
 import { createCacheWarmRouter, startCacheWarmer } from './routes/cache-warm.js';
 import { warmup, cleanup as cleanupFetcher } from '../core/fetcher.js';
+import { setExtractorRedis } from '../core/domain-extractors.js';
 import { registerPremiumHooks } from './premium/index.js';
 import { readFileSync } from 'fs';
 import { join, dirname } from 'path';
@@ -421,6 +422,29 @@ export function startServer(config = {}) {
     const port = config.port || parseInt(process.env.PORT || '3000', 10);
     // Activate premium strategy hooks (SWR cache, domain intelligence, race).
     registerPremiumHooks();
+    // Inject Redis into the domain extractor cache for cross-pod cache sharing.
+    // When REDIS_URL is set (multi-pod k8s deployments), all pods share one cache
+    // so the first pod to fetch a URL populates it for all others.
+    if (process.env.REDIS_URL) {
+        // @ts-ignore — ioredis CJS/ESM interop
+        import('ioredis').then((IoRedisModule) => {
+            const IoRedis = IoRedisModule.default ?? IoRedisModule;
+            const url = process.env.REDIS_URL;
+            const parsed = new URL(url);
+            const redis = new IoRedis({
+                host: parsed.hostname,
+                port: parseInt(parsed.port || '6379', 10),
+                db: parseInt(parsed.pathname?.slice(1) || '0', 10) || 0,
+                lazyConnect: true,
+                maxRetriesPerRequest: 3,
+                enableOfflineQueue: false,
+            });
+            setExtractorRedis(redis);
+            log.info('Redis extractor cache initialized (shared cross-pod cache active)');
+        }).catch((err) => {
+            log.warn('Failed to init Redis extractor cache (in-memory only)', { error: err.message });
+        });
+    }
     // Pre-warm browser resources in the background to reduce first-request latency.
     void warmup().catch((error) => {
         log.warn('Browser warmup failed', { error: error instanceof Error ? error.message : String(error) });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.66",
+  "version": "0.21.68",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",