webpeel 0.9.0 → 0.10.0

package/dist/core/hotel-search.js

@@ -0,0 +1,381 @@
+/**
+ * Hotel search module — searches multiple travel sites and returns sorted hotel listings.
+ *
+ * Sources: Kayak, Booking.com, Google Travel
+ * All sources are fetched in parallel; failures are captured per-source without
+ * crashing the overall search.
+ */
+import { peel } from '../index.js';
+import { extractListings } from './extract-listings.js';
+import { findSchemaForUrl, extractWithSchema } from './schema-extraction.js';
+// ── Date Parsing ──────────────────────────────────────────────────────────────
+/**
+ * Parse a date string (ISO or relative) into an ISO date string (YYYY-MM-DD).
+ *
+ * Supported relative formats:
+ *   - "tomorrow" → today + 1 day
+ *   - "next <weekday>" → next occurrence of that weekday
+ *   - ISO date "2026-02-20" → returned as-is
+ */
+export function parseDate(input, baseDate) {
+    const base = baseDate ?? new Date();
+    // Normalise
+    const normalised = input.trim().toLowerCase();
+    if (normalised === 'today') {
+        return toIsoDate(base);
+    }
+    if (normalised === 'tomorrow') {
+        const d = new Date(base);
+        d.setDate(d.getDate() + 1);
+        return toIsoDate(d);
+    }
+    // "next <weekday>"
+    const nextMatch = normalised.match(/^next\s+(monday|tuesday|wednesday|thursday|friday|saturday|sunday)$/);
+    if (nextMatch) {
+        const weekdays = ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'];
+        const targetDay = weekdays.indexOf(nextMatch[1]);
+        const d = new Date(base);
+        const currentDay = d.getDay();
+        let daysUntil = targetDay - currentDay;
+        if (daysUntil <= 0)
+            daysUntil += 7;
+        d.setDate(d.getDate() + daysUntil);
+        return toIsoDate(d);
+    }
+    // Try ISO date (YYYY-MM-DD)
+    if (/^\d{4}-\d{2}-\d{2}$/.test(input.trim())) {
+        return input.trim();
+    }
+    // Fallback: try to parse as a generic date string
+    const parsed = new Date(input);
+    if (!isNaN(parsed.getTime())) {
+        return toIsoDate(parsed);
+    }
+    throw new Error(`Unrecognized date format: "${input}"`);
+}
+function toIsoDate(d) {
+    const yyyy = d.getFullYear();
+    const mm = String(d.getMonth() + 1).padStart(2, '0');
+    const dd = String(d.getDate()).padStart(2, '0');
+    return `${yyyy}-${mm}-${dd}`;
+}
+/** Add N days to an ISO date string and return the new ISO date string. */
+export function addDays(isoDate, days) {
+    const d = new Date(isoDate + 'T12:00:00Z');
+    d.setUTCDate(d.getUTCDate() + days);
+    const yyyy = d.getUTCFullYear();
+    const mm = String(d.getUTCMonth() + 1).padStart(2, '0');
+    const dd = String(d.getUTCDate()).padStart(2, '0');
+    return `${yyyy}-${mm}-${dd}`;
+}
+// ── URL Builders ──────────────────────────────────────────────────────────────
+/**
+ * Convert a destination name to a Kayak-friendly slug.
+ * e.g. "Manhattan, New York" → "Manhattan,New-York"
+ * e.g. "Long Island City" → "Long-Island-City"
+ */
+export function toKayakSlug(destination) {
+    return destination
+        .split(',')
+        .map(part => part.trim().replace(/\s+/g, '-'))
+        .join(',');
+}
+/**
+ * Build the search URL for each source.
+ */
+export function buildSourceUrls(destination, checkin, checkout) {
+    const kayakSlug = toKayakSlug(destination);
+    const bookingDest = encodeURIComponent(destination);
+    const googleDest = destination.replace(/\s+/g, '+');
+    const expediaDest = encodeURIComponent(destination);
+    return [
+        {
+            name: 'kayak',
+            url: `https://www.kayak.com/hotels/${kayakSlug}/${checkin}/${checkout}?sort=price_a`,
+        },
+        {
+            name: 'booking',
+            url: `https://www.booking.com/searchresults.html?ss=${bookingDest}&checkin=${checkin}&checkout=${checkout}&order=price`,
+        },
+        {
+            name: 'google',
+            url: `https://www.google.com/travel/hotels/${googleDest}`,
+        },
+        {
+            name: 'expedia',
+            url: `https://www.expedia.com/Hotel-Search?destination=${expediaDest}&startDate=${checkin}&endDate=${checkout}&sort=PRICE_LOW_TO_HIGH`,
+        },
+    ];
+}
+// ── Price & Rating Parsers ────────────────────────────────────────────────────
+/**
+ * Parse a price display string into a numeric USD value.
+ * Returns null if unparseable.
+ *
+ * Examples:
+ *   "$119"     → 119
+ *   "$1,299"   → 1299
+ *   "£85"      → 85  (GBP treated as USD approximation)
+ *   "€95"      → 95
+ *   "US$200"   → 200
+ */
+export function parsePrice(raw) {
+    if (!raw)
+        return null;
+    // Remove currency symbols and "US$" prefix, commas, whitespace
+    const cleaned = raw.replace(/US\$|[$£€¥₹]/g, '').replace(/,/g, '').trim();
+    // Extract first number
+    const match = cleaned.match(/(\d+(?:\.\d+)?)/);
+    if (!match)
+        return null;
+    const n = parseFloat(match[1]);
+    return isNaN(n) ? null : n;
+}
+/**
+ * Parse a rating string into a numeric value.
+ * Returns null if unparseable.
+ *
+ * Examples:
+ *   "Scored 8.4"      → 8.4
+ *   "4.2/5"           → 4.2
+ *   "4.2/5 (1.4K)"    → 4.2
+ *   "8.3"             → 8.3
+ *   "Very Good 8.6"   → 8.6
+ */
+export function parseRating(raw) {
+    if (!raw)
+        return null;
+    // "Scored N.N" or "Very Good N.N" etc.
+    const scoredMatch = raw.match(/(\d+(?:\.\d+)?)\s*\/\s*\d/);
+    if (scoredMatch) {
+        const n = parseFloat(scoredMatch[1]);
+        return isNaN(n) ? null : n;
+    }
+    // Extract last number (handles "Scored 8.4", "Very Good 8.6", standalone "8.3")
+    const numMatch = raw.match(/(\d+(?:\.\d+)?)/g);
+    if (!numMatch)
+        return null;
+    // Take the last number that looks like a rating (0–10 scale or 0–5 scale)
+    for (let i = numMatch.length - 1; i >= 0; i--) {
+        const n = parseFloat(numMatch[i]);
+        if (!isNaN(n) && n >= 0 && n <= 10)
+            return n;
+    }
+    return null;
+}
+// ── Result Normalisation ──────────────────────────────────────────────────────
+/**
+ * Map an extracted listing item to a HotelResult, tagged with the source name.
+ */
+function normaliseToHotelResult(item, sourceName) {
+    const name = item.title?.trim();
+    if (!name)
+        return null;
+    const priceDisplay = item.price ?? '';
+    const ratingDisplay = item.rating ?? '';
+    return {
+        name,
+        price: parsePrice(priceDisplay),
+        priceDisplay,
+        rating: parseRating(ratingDisplay),
+        ratingDisplay,
+        source: sourceName,
+        link: item.link ?? '',
+        location: item.description?.trim() || undefined,
+        image: item.image || undefined,
+    };
+}
+// ── Deduplication ─────────────────────────────────────────────────────────────
+/**
+ * Deduplicate hotel results by name (case-insensitive).
+ * When duplicates exist, keep the one with the most data (price + rating),
+ * with lowest price as a tiebreaker.
+ */
+export function deduplicateHotels(hotels) {
+    const byName = new Map();
+    for (const hotel of hotels) {
+        const key = hotel.name.toLowerCase().replace(/\s+/g, ' ').trim();
+        const existing = byName.get(key);
+        if (!existing) {
+            byName.set(key, hotel);
+            continue;
+        }
+        // Score = number of non-null data fields
+        const scoreNew = (hotel.price !== null ? 1 : 0) + (hotel.rating !== null ? 1 : 0);
+        const scoreOld = (existing.price !== null ? 1 : 0) + (existing.rating !== null ? 1 : 0);
+        if (scoreNew > scoreOld) {
+            byName.set(key, hotel);
+        }
+        else if (scoreNew === scoreOld) {
+            // Tiebreak: prefer the one with lower price (or keep existing if equal)
+            if (hotel.price !== null && (existing.price === null || hotel.price < existing.price)) {
+                byName.set(key, hotel);
+            }
+        }
+    }
+    return Array.from(byName.values());
+}
+// ── Sorting ───────────────────────────────────────────────────────────────────
+/**
+ * Sort hotel results.
+ * - price: ascending, nulls last
+ * - rating: descending, nulls last
+ * - value: rating/price ratio, descending, nulls last
+ */
+export function sortHotels(hotels, sort) {
+    const sorted = [...hotels];
+    switch (sort) {
+        case 'price':
+            sorted.sort((a, b) => {
+                if (a.price === null && b.price === null)
+                    return 0;
+                if (a.price === null)
+                    return 1;
+                if (b.price === null)
+                    return -1;
+                return a.price - b.price;
+            });
+            break;
+        case 'rating':
+            sorted.sort((a, b) => {
+                if (a.rating === null && b.rating === null)
+                    return 0;
+                if (a.rating === null)
+                    return 1;
+                if (b.rating === null)
+                    return -1;
+                return b.rating - a.rating;
+            });
+            break;
+        case 'value': {
+            const valueOf = (h) => {
+                if (h.price === null || h.price === 0 || h.rating === null)
+                    return null;
+                return h.rating / h.price;
+            };
+            sorted.sort((a, b) => {
+                const va = valueOf(a);
+                const vb = valueOf(b);
+                if (va === null && vb === null)
+                    return 0;
+                if (va === null)
+                    return 1;
+                if (vb === null)
+                    return -1;
+                return vb - va;
+            });
+            break;
+        }
+    }
+    return sorted;
+}
+// ── Main Function ─────────────────────────────────────────────────────────────
+const DEFAULT_SOURCES = ['kayak', 'booking', 'google', 'expedia'];
+const SIMPLE_TIMEOUT = 15_000;
+const BROWSER_TIMEOUT = 30_000;
+const EXPEDIA_TIMEOUT = 60_000;
+/**
+ * Search multiple travel sites for hotels and return sorted, deduplicated results.
+ */
+export async function searchHotels(options) {
+    const startTime = Date.now();
+    // ── Parse dates ────────────────────────────────────────────────────────────
+    const checkin = parseDate(options.checkin);
+    const rawCheckout = options.checkout;
+    const checkout = rawCheckout ? parseDate(rawCheckout) : addDays(checkin, 1);
+    const destination = options.destination;
+    const sort = options.sort ?? 'price';
+    const limit = options.limit ?? 20;
+    const allowedSources = new Set((options.sources ?? DEFAULT_SOURCES).map(s => s.toLowerCase()));
+    const useGlobalStealth = options.stealth ?? false;
+    // ── Build source URLs ──────────────────────────────────────────────────────
+    const allSourceUrls = buildSourceUrls(destination, checkin, checkout).filter(s => allowedSources.has(s.name));
+    // ── Fetch all sources in parallel ──────────────────────────────────────────
+    const settled = await Promise.allSettled(allSourceUrls.map(async (src) => {
+        const isKayak = src.name === 'kayak';
+        const isBooking = src.name === 'booking';
+        const isExpedia = src.name === 'expedia';
+        const useStealth = useGlobalStealth || isKayak || isExpedia;
+        const useRender = useStealth || isBooking;
+        const timeout = isExpedia ? EXPEDIA_TIMEOUT : (useRender ? BROWSER_TIMEOUT : SIMPLE_TIMEOUT);
+        // Expedia is a SPA — wait for property listings to appear before extracting
+        const actions = isExpedia
+            ? [{ type: 'waitForSelector', selector: "[data-stid='property-listing'], li.uitk-spacing" }]
+            : undefined;
+        const result = await peel(src.url, {
+            format: 'html',
+            render: useRender,
+            stealth: useStealth,
+            timeout,
+            ...(actions ? { actions } : {}),
+        });
+        // Prefer CSS schema extraction when a schema is available for this source
+        const schema = findSchemaForUrl(src.url);
+        const hotels = [];
+        if (schema) {
+            const schemaItems = extractWithSchema(result.content, schema, src.url);
+            for (const item of schemaItems) {
+                const mapped = {
+                    title: typeof item.title === 'string' ? item.title : undefined,
+                    price: typeof item.price === 'string' ? item.price : undefined,
+                    rating: typeof item.rating === 'string' ? item.rating : undefined,
+                    link: typeof item.link === 'string' ? item.link : undefined,
+                    image: typeof item.image === 'string' ? item.image : undefined,
+                    description: typeof item.location === 'string' ? item.location : undefined,
+                };
+                const hotel = normaliseToHotelResult(mapped, src.name);
+                if (hotel)
+                    hotels.push(hotel);
+            }
+        }
+        // Fall back to generic extraction if schema yielded nothing
+        if (hotels.length === 0) {
+            const listings = extractListings(result.content, src.url);
+            for (const item of listings) {
+                const hotel = normaliseToHotelResult(item, src.name);
+                if (hotel)
+                    hotels.push(hotel);
+            }
+        }
+        return { name: src.name, hotels };
+    }));
+    // ── Collect per-source status and results ──────────────────────────────────
+    const sourceStats = [];
+    const allHotels = [];
+    for (let i = 0; i < allSourceUrls.length; i++) {
+        const src = allSourceUrls[i];
+        const outcome = settled[i];
+        if (outcome.status === 'fulfilled') {
+            const { hotels } = outcome.value;
+            sourceStats.push({ name: src.name, count: hotels.length, status: 'ok' });
+            allHotels.push(...hotels);
+        }
+        else {
+            const errMsg = outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason);
+            const isBlocked = errMsg.toLowerCase().includes('blocked') ||
+                errMsg.toLowerCase().includes('403') ||
+                errMsg.toLowerCase().includes('cloudflare');
+            sourceStats.push({
+                name: src.name,
+                count: 0,
+                status: isBlocked ? 'blocked' : 'error',
+                error: errMsg,
+            });
+        }
+    }
+    // ── Deduplicate, sort, limit ───────────────────────────────────────────────
+    const unique = deduplicateHotels(allHotels);
+    const sorted = sortHotels(unique, sort);
+    const results = sorted.slice(0, limit);
+    const elapsed = Date.now() - startTime;
+    return {
+        destination,
+        checkin,
+        checkout,
+        totalResults: results.length,
+        results,
+        sources: sourceStats,
+        elapsed,
+    };
+}
+//# sourceMappingURL=hotel-search.js.map

package/dist/core/hotel-search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hotel-search.js","sourceRoot":"","sources":["../../src/core/hotel-search.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAkD7E,iFAAiF;AAEjF;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CAAC,KAAa,EAAE,QAAe;IACtD,MAAM,IAAI,GAAG,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC;IAEpC,YAAY;IACZ,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE9C,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;QAC3B,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAED,IAAI,UAAU,KAAK,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3B,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAC;IAC1G,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChG,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAC;QAClD,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;QACzB,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;QAC9B,IAAI,SAAS,GAAG,SAAS,GAAG,UAAU,CAAC;QACvC,IAAI,SAAS,IAAI,CAAC;YAAE,SAAS,IAAI,CAAC,CAAC;QACnC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,CAAC;QACnC,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IAED,4BAA4B;IAC5B,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAC7C,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;IACtB,CAAC;IAED,kDAAkD;IAClD,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;QAC7B,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,GAAG,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,SAAS,CAAC,CAAO;IACxB,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7B,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACrD,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAChD,OAAO,GAAG,IAAI,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;AAC/B,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAY;IACnD,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,OAAO,GAAG,YAAY,CAAC,CAAC;IAC3C,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,CAAC;IAChC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACxD,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACnD,OAAO,GAAG,IAAI,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;AAC/B,CAAC;AAED,iFAAiF;AAEjF;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,WAAmB;IAC7C,OAAO,WAAW;SACf,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;SAC7C,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC;AAOD;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,WAAmB,EACnB,OAAe,EACf,QAAgB;IAEhB,MAAM,SAAS,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IACpD,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAEpD,MAAM,WAAW,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAEpD,OAAO;QACL;YACE,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,gCAAgC,SAAS,IAAI,OAAO,IAAI,QAAQ,eAAe;SACrF;QACD;YACE,IAAI,EAAE,SAAS;YACf,GAAG,EAAE,iDAAiD,WAAW,YAAY,OAAO,aAAa,QAAQ,cAAc;SACxH;QACD;YACE,IAAI,EAAE,QAAQ;YACd,GAAG,EAAE,wCAAwC,UAAU,EAAE;SAC1D;QACD;YACE,IAAI,EAAE,SAAS;YACf,GAAG,EAAE,oDAAoD,WAAW,cAAc,OAAO,YAAY,QAAQ,yBAAyB;SACvI;KACF,CAAC;AACJ,CAAC;AAED,iFAAiF;AAEjF;;;;;;;;;;GAUG;AACH,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,+DAA+D;IAC/D,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1E,uBAAuB;IACvB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC/C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;IAChC,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7B,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,uCAAuC;IACvC,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC3D,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,CAAC,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,CAAC;QACtC,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,gFAAgF;IAChF,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAC/C,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,0EAA0E;IAC1E,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,iFAAiF;AAEjF;;GAEG;AACH,SAAS,sBAAsB,CAC7B,IAAiJ,EACjJ,UAAkB;IAElB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;IACtC,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;IAExC,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,UAAU,CAAC,YAAY,CAAC;QAC/B,YAAY;QACZ,MAAM,EAAE,WAAW,CAAC,aAAa,CAAC;QAClC,aAAa;QACb,MAAM,EAAE,UAAU;QAClB,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;QACrB,QAAQ,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,SAAS;QAC/C,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,SAAS;KAC/B,CAAC;AACJ,CAAC;AAED,iFAAiF;AAEjF;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAqB;IACrD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;IAE9C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACjE,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACvB,SAAS;QACX,CAAC;QAED,yCAAyC;QACzC,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClF,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAExF,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACzB,CAAC;aAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,wEAAwE;YACxE,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtF,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;AACrC,CAAC;AAED,iFAAiF;AAEjF;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,MAAqB,EAAE,IAAkC;IAClF,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAE3B,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,OAAO;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACnB,IAAI,CAAC,CAAC,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,KAAK,KAAK,IAAI;oBAAE,OAAO,CAAC,CAAC;gBACnD,IAAI,CAAC,CAAC,KAAK,KAAK,IAAI;oBAAE,OAAO,CAAC,CAAC;gBAC/B,IAAI,CAAC,CAAC,KAAK,KAAK,IAAI;oBAAE,OAAO,CAAC,CAAC,CAAC;gBAChC,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;YAC3B,CAAC,CAAC,CAAC;YACH,MAAM;QAER,KAAK,QAAQ;YACX,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACnB,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI;oBAAE,OAAO,CAAC,CAAC;gBACrD,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI;oBAAE,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI;oBAAE,OAAO,CAAC,CAAC,CAAC;gBACjC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;YAC7B,CAAC,CAAC,CAAC;YACH,MAAM;QAER,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,OAAO,GAAG,CAAC,CAAc,EAAiB,EAAE;gBAChD,IAAI,CAAC,CAAC,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI;oBAAE,OAAO,IAAI,CAAC;gBACxE,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC;YAC5B,CAAC,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACnB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;gBACtB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;gBACtB,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI;oBAAE,OAAO,CAAC,CAAC;gBACzC,IAAI,EAAE,KAAK,IAAI;oBAAE,OAAO,CAAC,CAAC;gBAC1B,IAAI,EAAE,KAAK,IAAI;oBAAE,OAAO,CAAC,CAAC,CAAC;gBAC3B,OAAO,EAAE,GAAG,EAAE,CAAC;YACjB,CAAC,CAAC,CAAC;YACH,MAAM;QACR,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,iFAAiF;AAEjF,MAAM,eAAe,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;AAClE,MAAM,cAAc,GAAG,MAAM,CAAC;AAC9B,MAAM,eAAe,GAAG,MAAM,CAAC;AAC/B,MAAM,eAAe,GAAG,MAAM,CAAC;AAE/B;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA2B;IAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,8EAA8E;IAC9E,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC;IACrC,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAE5E,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IACxC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC;IACrC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAClC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAC/F,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IAElD,8EAA8E;IAC9E,MAAM,aAAa,GAAG,eAAe,CAAC,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAC/E,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAC3B,CAAC;IAEF,8EAA8E;IAC9E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAC9B,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,KAAK,OAAO,CAAC;QACrC,MAAM,SAAS,GAAG,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC;QACzC,MAAM,SAAS,GAAG,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC;QAEzC,MAAM,UAAU,GAAG,gBAAgB,IAAI,OAAO,IAAI,SAAS,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU,IAAI,SAAS,CAAC;QAC1C,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;QAE7F,4EAA4E;QAC5E,MAAM,OAAO,GAA6B,SAAS;YACjD,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,iDAAiD,EAAE,CAAC;YAC5F,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE;YACjC,MAAM,EAAE,MAAM;YACd,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,UAAU;YACnB,OAAO;YACP,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChC,CAAC,CAAC;QAEH,0EAA0E;QAC1E,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,MAAM,GAAkB,EAAE,CAAC;QAEjC,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;YACvE,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;gBAC/B,MAAM,MAAM,GAAG;oBACb,KAAK,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;oBAC9D,KAAK,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;oBAC9D,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;oBACjE,IAAI,EAAE,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;oBAC3D,KAAK,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;oBAC9D,WAAW,EAAE,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;iBAC3E,CAAC;gBACF,MAAM,KAAK,GAAG,sBAAsB,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;gBACvD,IAAI,KAAK;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,4DAA4D;QAC5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;YAC1D,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;gBACrD,IAAI,KAAK;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IACpC,CAAC,CAAC,CACH,CAAC;IAEF,8EAA8E;IAC9E,MAAM,WAAW,GAAiC,EAAE,CAAC;IACrD,MAAM,SAAS,GAAkB,EAAE,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,GAAG,GAAG,aAAa,CAAC,CAAC,CAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;QAE5B,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YACnC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;YACjC,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACzE,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACjG,MAAM,SAAS,GACb,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC;gBACxC,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACpC,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YAC9C,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO;gBACvC,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAEvC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAEvC,OAAO;QACL,WAAW;QACX,OAAO;QACP,QAAQ;QACR,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,OAAO;QACP,OAAO,EAAE,WAAW;QACpB,OAAO;KACR,CAAC;AACJ,CAAC"}

package/dist/core/llm-extract.d.ts

@@ -0,0 +1,42 @@
+/**
+ * LLM-based extraction: sends markdown/text content to an LLM
+ * with instructions to extract structured data.
+ *
+ * Supports OpenAI-compatible APIs (OpenAI, Anthropic via proxy, local models).
+ */
+export interface LLMExtractionOptions {
+    content: string;
+    instruction?: string;
+    schema?: object;
+    apiKey?: string;
+    baseUrl?: string;
+    model?: string;
+    maxTokens?: number;
+}
+export interface LLMExtractionResult {
+    items: Array<Record<string, any>>;
+    tokensUsed: {
+        input: number;
+        output: number;
+    };
+    model: string;
+    cost?: number;
+}
+/**
+ * Build the user message from content + optional instruction + optional schema.
+ */
+export declare function buildUserMessage(content: string, instruction?: string, schema?: object): string;
+/**
+ * Calculate estimated cost in USD for a given model and token counts.
+ */
+export declare function estimateCost(model: string, inputTokens: number, outputTokens: number): number | undefined;
+/**
+ * Parse the LLM response text into an items array.
+ * Handles both `{ "items": [...] }` and `[...]` formats.
+ */
+export declare function parseItems(text: string): Array<Record<string, any>>;
+/**
+ * Extract structured data from content using an LLM.
+ */
+export declare function extractWithLLM(options: LLMExtractionOptions): Promise<LLMExtractionResult>;
+//# sourceMappingURL=llm-extract.d.ts.map

package/dist/core/llm-extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-extract.d.ts","sourceRoot":"","sources":["../../src/core/llm-extract.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;IAClC,UAAU,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAcD;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAe/F;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMzG;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAqCnE;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAsEhG"}

package/dist/core/llm-extract.js

@@ -0,0 +1,144 @@
+/**
+ * LLM-based extraction: sends markdown/text content to an LLM
+ * with instructions to extract structured data.
+ *
+ * Supports OpenAI-compatible APIs (OpenAI, Anthropic via proxy, local models).
+ */
+// Cost per 1M tokens (input, output) for known models
+const MODEL_COSTS = {
+    'gpt-4o-mini': [0.15, 0.60],
+    'gpt-4o': [2.50, 10.0],
+};
+const SYSTEM_PROMPT = `You are a data extraction assistant. Extract structured data from the provided web content.
+Return a JSON array of objects. Each object represents one item/listing found on the page.
+Always include these fields when available: title, price, link, rating, description, image.
+If the user provides additional instructions, follow them.
+Return ONLY valid JSON — no markdown, no explanation, just the array.`;
+/**
+ * Build the user message from content + optional instruction + optional schema.
+ */
+export function buildUserMessage(content, instruction, schema) {
+    // Truncate content if over 100K chars
+    const truncated = content.length > 100_000 ? content.slice(0, 50_000) : content;
+    let msg = `Here is the web content to extract data from:\n\n${truncated}`;
+    if (schema) {
+        msg += `\n\nExtract data matching this schema: ${JSON.stringify(schema, null, 2)}`;
+    }
+    if (instruction) {
+        msg += `\n\nAdditional instruction: ${instruction}`;
+    }
+    return msg;
+}
+/**
+ * Calculate estimated cost in USD for a given model and token counts.
+ */
+export function estimateCost(model, inputTokens, outputTokens) {
+    // Normalize model key (strip version suffixes like -2024-11-20 for matching)
+    const key = Object.keys(MODEL_COSTS).find(k => model.startsWith(k) || model === k);
+    if (!key)
+        return undefined;
+    const [inputRate, outputRate] = MODEL_COSTS[key];
+    return (inputTokens / 1_000_000) * inputRate + (outputTokens / 1_000_000) * outputRate;
+}
+/**
+ * Parse the LLM response text into an items array.
+ * Handles both `{ "items": [...] }` and `[...]` formats.
+ */
+export function parseItems(text) {
+    const trimmed = text.trim();
+    // Try to parse as-is first
+    let parsed;
+    try {
+        parsed = JSON.parse(trimmed);
+    }
+    catch {
+        // Try to extract JSON from the text (sometimes LLMs add preamble despite instructions)
+        const arrayMatch = trimmed.match(/\[[\s\S]*\]/);
+        const objMatch = trimmed.match(/\{[\s\S]*\}/);
+        if (arrayMatch) {
+            try {
+                parsed = JSON.parse(arrayMatch[0]);
+            }
+            catch { /* fall through */ }
+        }
+        else if (objMatch) {
+            try {
+                parsed = JSON.parse(objMatch[0]);
+            }
+            catch { /* fall through */ }
+        }
+        if (parsed === undefined) {
+            throw new Error(`Failed to parse LLM response as JSON: ${trimmed.slice(0, 200)}`);
+        }
+    }
+    // Handle { items: [...] } or { data: [...] } or { results: [...] }
+    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+        const obj = parsed;
+        if (Array.isArray(obj['items']))
+            return obj['items'];
+        if (Array.isArray(obj['data']))
+            return obj['data'];
+        if (Array.isArray(obj['results']))
+            return obj['results'];
+        // Single object — wrap in array
+        return [obj];
+    }
+    // Handle bare array
+    if (Array.isArray(parsed)) {
+        return parsed;
+    }
+    return [];
+}
+/**
+ * Extract structured data from content using an LLM.
+ */
+export async function extractWithLLM(options) {
+    const { content, instruction, schema, baseUrl = 'https://api.openai.com/v1', model = 'gpt-4o-mini', maxTokens = 4000, } = options;
+    const apiKey = options.apiKey || process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+        throw new Error('LLM extraction requires an API key.\n' +
+            'Set OPENAI_API_KEY environment variable or use --llm-key <key>');
+    }
+    const userMessage = buildUserMessage(content, instruction, schema);
+    const response = await fetch(`${baseUrl}/chat/completions`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${apiKey}`,
+        },
+        body: JSON.stringify({
+            model,
+            messages: [
+                { role: 'system', content: SYSTEM_PROMPT },
+                { role: 'user', content: userMessage },
+            ],
+            temperature: 0,
+            max_tokens: maxTokens,
+            response_format: { type: 'json_object' },
+        }),
+    });
+    if (!response.ok) {
+        const body = await response.text().catch(() => '');
+        if (response.status === 401) {
+            throw new Error(`LLM API authentication failed (401). Check your API key.`);
+        }
+        if (response.status === 429) {
+            throw new Error(`LLM API rate limit exceeded (429). Please wait and retry.`);
+        }
+        throw new Error(`LLM API error: HTTP ${response.status}${body ? ` — ${body.slice(0, 200)}` : ''}`);
+    }
+    const data = await response.json();
+    const rawText = data.choices?.[0]?.message?.content ?? '';
+    const items = parseItems(rawText);
+    const inputTokens = data.usage?.prompt_tokens ?? 0;
+    const outputTokens = data.usage?.completion_tokens ?? 0;
+    const resolvedModel = data.model ?? model;
+    const cost = estimateCost(resolvedModel, inputTokens, outputTokens);
+    return {
+        items,
+        tokensUsed: { input: inputTokens, output: outputTokens },
+        model: resolvedModel,
+        cost,
+    };
+}
+//# sourceMappingURL=llm-extract.js.map

package/dist/core/llm-extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-extract.js","sourceRoot":"","sources":["../../src/core/llm-extract.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAmBH,sDAAsD;AACtD,MAAM,WAAW,GAAqC;IACpD,aAAa,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;IAC3B,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;CACvB,CAAC;AAEF,MAAM,aAAa,GAAG;;;;sEAIgD,CAAC;AAEvE;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,WAAoB,EAAE,MAAe;IACrF,sCAAsC;IACtC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAEhF,IAAI,GAAG,GAAG,oDAAoD,SAAS,EAAE,CAAC;IAE1E,IAAI,MAAM,EAAE,CAAC;QACX,GAAG,IAAI,0CAA0C,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;IACrF,CAAC;IAED,IAAI,WAAW,EAAE,CAAC;QAChB,GAAG,IAAI,+BAA+B,WAAW,EAAE,CAAC;IACtD,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,WAAmB,EAAE,YAAoB;IACnF,6EAA6E;IAC7E,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC;IACnF,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,GAAG,WAAW,CAAC,GAAG,CAAE,CAAC;IAClD,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,SAAS,GAAG,CAAC,YAAY,GAAG,SAAS,CAAC,GAAG,UAAU,CAAC;AACzF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE5B,2BAA2B;IAC3B,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,uFAAuF;QACvF,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC9C,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC;QAC1E,CAAC;aAAM,IAAI,QAAQ,EAAE,CAAC;YACpB,IAAI,CAAC;gBAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAED,mEAAmE;IACnE,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACnE,MAAM,GAAG,GAAG,MAA6B,CAAC;QAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAAE,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAAE,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC;QACnD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAAE,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC;QACzD,gCAAgC;QAChC,OAAO,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;IAED,oBAAoB;IACpB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA6B;IAChE,MAAM,EACJ,OAAO,EACP,WAAW,EACX,MAAM,EACN,OAAO,GAAG,2BAA2B,EACrC,KAAK,GAAG,aAAa,EACrB,SAAS,GAAG,IAAI,GACjB,GAAG,OAAO,CAAC;IAEZ,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAE5D,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,uCAAuC;YACvC,gEAAgE,CACjE,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAEnE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,mBAAmB,EAAE;QAC1D,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,eAAe,EAAE,UAAU,MAAM,EAAE;SACpC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;gBAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;aACvC;YACD,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,SAAS;YACrB,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;SACzC,CAAC;KACH,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC/E,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrG,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAI/B,CAAC;IAEF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;IAC1D,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;IAElC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,CAAC,CAAC;IACnD,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,EAAE,iBAAiB,IAAI,CAAC,CAAC;IACxD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC;IAC1C,MAAM,IAAI,GAAG,YAAY,CAAC,aAAa,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;IAEpE,OAAO;QACL,KAAK;QACL,UAAU,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE;QACxD,KAAK,EAAE,aAAa;QACpB,IAAI;KACL,CAAC;AACJ,CAAC"}

package/dist/core/profiles.d.ts

@@ -0,0 +1,48 @@
+/**
+ * WebPeel Profile Management
+ *
+ * Manages named browser profiles stored in ~/.webpeel/profiles/<name>/
+ * Each profile contains:
+ *   - storage-state.json  (Playwright storage state: cookies, localStorage, origins)
+ *   - metadata.json       (name, created, lastUsed, domains, description)
+ */
+export interface ProfileMetadata {
+    name: string;
+    created: string;
+    lastUsed: string;
+    domains: string[];
+    description?: string;
+}
+/**
+ * Valid profile names: letters, digits, hyphens only. No spaces or special chars.
+ */
+export declare function isValidProfileName(name: string): boolean;
+/**
+ * Get the directory path for a named profile, or null if it doesn't exist.
+ */
+export declare function getProfilePath(name: string): string | null;
+/**
+ * Load the Playwright storage state (cookies + localStorage) for a named profile.
+ * Returns null if the profile or storage-state.json doesn't exist.
+ */
+export declare function loadStorageState(name: string): any | null;
+/**
+ * Update the lastUsed timestamp for a profile.
+ */
+export declare function touchProfile(name: string): void;
+/**
+ * List all profiles, sorted by lastUsed descending.
+ */
+export declare function listProfiles(): ProfileMetadata[];
+/**
+ * Delete a named profile. Returns true if deleted, false if not found.
+ */
+export declare function deleteProfile(name: string): boolean;
+/**
+ * Interactively create a new profile:
+ * 1. Launches a VISIBLE (headed) Chromium browser
+ * 2. User navigates and logs into sites
+ * 3. On browser close or Ctrl+C, captures storage state and saves the profile
+ */
+export declare function createProfile(name: string, description?: string): Promise<void>;
+//# sourceMappingURL=profiles.d.ts.map

package/dist/core/profiles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.d.ts","sourceRoot":"","sources":["../../src/core/profiles.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAgBH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAcD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAExD;AAID;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM1D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,GAAG,IAAI,CAQzD;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAU/C;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,eAAe,EAAE,CAsBhD;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CASnD;AAID;;;;;GAKG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAqHrF"}